From 3db36ea688fd00bce3236a1acf0ccb4a559fbf0d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 20 Dec 2021 19:18:22 +0100 Subject: [PATCH 001/206] feat(api): search sessions support AND|THEN|OR between events feat(api): search sessions support single-operator-multiple-values --- api/chalicelib/core/events.py | 35 ++++----- api/chalicelib/core/sessions.py | 124 ++++++++++++++++++++++---------- api/chalicelib/utils/helper.py | 72 ++++++++++++------- api/schemas.py | 46 +++++++++++- 4 files changed, 192 insertions(+), 85 deletions(-) diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index 69213a079..deb16cb8b 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -1,6 +1,7 @@ -from chalicelib.utils import pg_client, helper -from chalicelib.core import sessions_metas, metadata +import schemas from chalicelib.core import issues +from chalicelib.core import sessions_metas, metadata +from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC from chalicelib.utils.event_filter_definition import SupportedFilter, Event @@ -235,23 +236,23 @@ def __generic_autocomplete(event: Event): class event_type: - CLICK = Event(ui_type="CLICK", table="events.clicks", column="label") - INPUT = Event(ui_type="INPUT", table="events.inputs", column="label") - LOCATION = Event(ui_type="LOCATION", table="events.pages", column="base_path") - CUSTOM = Event(ui_type="CUSTOM", table="events_common.customs", column="name") - REQUEST = Event(ui_type="REQUEST", table="events_common.requests", column="url") - GRAPHQL = Event(ui_type="GRAPHQL", table="events.graphql", column="name") - STATEACTION = Event(ui_type="STATEACTION", table="events.state_actions", column="name") - ERROR = Event(ui_type="ERROR", table="events.errors", + CLICK = Event(ui_type=schemas.EventType.click, table="events.clicks", column="label") + INPUT = Event(ui_type=schemas.EventType.input, table="events.inputs", column="label") + LOCATION = Event(ui_type=schemas.EventType.location, table="events.pages", column="base_path") + CUSTOM = Event(ui_type=schemas.EventType.custom, table="events_common.customs", column="name") + REQUEST = Event(ui_type=schemas.EventType.request, table="events_common.requests", column="url") + GRAPHQL = Event(ui_type=schemas.EventType.graphql, table="events.graphql", column="name") + STATEACTION = Event(ui_type=schemas.EventType.state_action, table="events.state_actions", column="name") + ERROR = Event(ui_type=schemas.EventType.error, table="events.errors", column=None) # column=None because errors are searched by name or message - METADATA = Event(ui_type="METADATA", table="public.sessions", column=None) + METADATA = Event(ui_type=schemas.EventType.metadata, table="public.sessions", column=None) # IOS - CLICK_IOS = Event(ui_type="CLICK_IOS", table="events_ios.clicks", column="label") - INPUT_IOS = Event(ui_type="INPUT_IOS", table="events_ios.inputs", column="label") - VIEW_IOS = Event(ui_type="VIEW_IOS", table="events_ios.views", column="name") - CUSTOM_IOS = Event(ui_type="CUSTOM_IOS", table="events_common.customs", column="name") - REQUEST_IOS = Event(ui_type="REQUEST_IOS", table="events_common.requests", column="url") - ERROR_IOS = Event(ui_type="ERROR_IOS", table="events_ios.crashes", + CLICK_IOS = Event(ui_type=schemas.EventType.click_ios, table="events_ios.clicks", column="label") + INPUT_IOS = Event(ui_type=schemas.EventType.input_ios, table="events_ios.inputs", column="label") + VIEW_IOS = Event(ui_type=schemas.EventType.view_ios, table="events_ios.views", column="name") + CUSTOM_IOS = Event(ui_type=schemas.EventType.custom_ios, table="events_common.customs", column="name") + REQUEST_IOS = Event(ui_type=schemas.EventType.request_ios, table="events_common.requests", column="url") + ERROR_IOS = Event(ui_type=schemas.EventType.error_ios, table="events_ios.crashes", column=None) # column=None because errors are searched by name or message diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index a6d5a50e0..ee732c848 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -109,7 +109,6 @@ def __is_multivalue(op: schemas.SearchEventOperator): def __get_sql_operator(op: schemas.SearchEventOperator): - op = op.lower() return { schemas.SearchEventOperator._is: "=", schemas.SearchEventOperator._is_any: "IN", @@ -145,6 +144,22 @@ def __get_sql_value_multiple(values): return tuple(values) if isinstance(values, list) else (values,) +def __multiple_conditions(condition, values, value_key="value"): + query = [] + for i in range(len(values)): + k = f"{value_key}_{i}" + query.append(condition.replace("value", k)) + return "(" + " OR ".join(query) + ")" + + +def __multiple_values(values, value_key="value"): + query_values = {} + for i in range(len(values)): + k = f"{value_key}_{i}" + query_values[k] = values[i] + return query_values + + @dev.timed def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, favorite_only=False, errors_only=False, error_status="ALL", @@ -257,15 +272,17 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints = [s.decode('UTF-8') for s in ss_constraints] events_query_from = [] event_index = 0 - + events_joiner = " FULL JOIN " if data.events_order == schemas.SearchEventOrder._or else " INNER JOIN LATERAL " for event in data.events: event_type = event.type.upper() + if not isinstance(event.value, list): + event.value = [event.value] op = __get_sql_operator(event.operator) is_not = False if __is_negation_operator(event.operator): is_not = True op = __reverse_sql_operator(op) - if event_index == 0: + if event_index == 0 or data.events_order == schemas.SearchEventOrder._or: event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", @@ -273,13 +290,12 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: event_from = "%s" event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", - f"event_{event_index - 1}.timestamp <= main.timestamp", "main.session_id=event_0.session_id"] - if __is_multivalue(event.operator): - event_args = {"value": __get_sql_value_multiple(event.value)} - else: - event.value = helper.string_to_op(value=event.value, op=event.operator) - event_args = {"value": helper.string_to_sql_like_with_op(event.value, op)} + if data.events_order == schemas.SearchEventOrder._then: + event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") + + event.value = helper.values_for_operator(value=event.value, op=event.operator) + event_args = __multiple_values(event.value) if event_type not in list(events.SUPPORTED_TYPES.keys()) \ or event.value in [None, "", "*"] \ and (event_type != events.event_type.ERROR.ui_type \ @@ -287,32 +303,50 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f continue if event_type == events.event_type.CLICK.ui_type: event_from = event_from % f"{events.event_type.CLICK.table} AS main " - event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") elif event_type == events.event_type.INPUT.ui_type: event_from = event_from % f"{events.event_type.INPUT.table} AS main " - event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") if len(event.custom) > 0: - event_where.append("main.value ILIKE %(custom)s") - event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") + event_where.append(__multiple_conditions(f"main.value ILIKE %(custom)s", + event.custom, value_key="custom")) + event_args = {**event_args, **__multiple_values(event.custom, value_key="custom")} + # event_where.append("main.value ILIKE %(custom)s") + # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") elif event_type == events.event_type.CUSTOM.ui_type: event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " - event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") elif event_type == events.event_type.REQUEST.ui_type: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") elif event_type == events.event_type.GRAPHQL.ui_type: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " - event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") + event_where.append(__multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") elif event_type == events.event_type.STATEACTION.ui_type: event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " - event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") elif event_type == events.event_type.ERROR.ui_type: - if event.source in [None, "*", ""]: - event.source = "js_exception" + # if event.source in [None, "*", ""]: + # event.source = "js_exception" event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" if event.value not in [None, "*", ""]: event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)") @@ -326,40 +360,55 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ----- IOS elif event_type == events.event_type.CLICK_IOS.ui_type: event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " - event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s", event.value)) + # event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") elif event_type == events.event_type.INPUT_IOS.ui_type: event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " - event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") - + event_where.append( + __multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s", event.value)) + # event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") if len(event.custom) > 0: - event_where.append("main.value ILIKE %(custom)s") - event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") + event_where.append(__multiple_conditions("main.value ILIKE %(custom)s", event.custom)) + event_args = {**event_args, **__multiple_values(event.custom, "custom")} + # event_where.append("main.value ILIKE %(custom)s") + # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " - event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s", event.value)) + # event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") elif event_type == events.event_type.CUSTOM_IOS.ui_type: event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " - event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") elif event_type == events.event_type.REQUEST_IOS.ui_type: event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " - event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") + event_where.append( + __multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s", + event.value)) + # event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") elif event_type == events.event_type.ERROR_IOS.ui_type: event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" if event.value not in [None, "*", ""]: - event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)") + event_where.append( + __multiple_conditions(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)", + event.value)) + # event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)") else: continue - if event_index == 0: + if event_index == 0 or data.events_order == schemas.SearchEventOrder._or: event_where += ss_constraints if is_not: if event_index == 0: events_query_from.append(cur.mogrify(f"""\ (SELECT session_id, - 0 AS timestamp, - {event_index} AS funnel_step + 0 AS timestamp FROM sessions WHERE EXISTS(SELECT session_id FROM {event_from} @@ -375,14 +424,13 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f events_query_from.append(cur.mogrify(f"""\ (SELECT event_0.session_id, - event_{event_index - 1}.timestamp AS timestamp, - {event_index} AS funnel_step + event_{event_index - 1}.timestamp AS timestamp WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ """, {**generic_args, **event_args}).decode('UTF-8')) else: events_query_from.append(cur.mogrify(f"""\ - (SELECT main.session_id, MIN(timestamp) AS timestamp,{event_index} AS funnel_step + (SELECT main.session_id, MIN(timestamp) AS timestamp FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY 1 @@ -394,7 +442,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_0.session_id, MIN(event_0.timestamp) AS first_event_ts, MAX(event_{event_index - 1}.timestamp) AS last_event_ts - FROM {(" INNER JOIN LATERAL ").join(events_query_from)} + FROM {events_joiner.join(events_query_from)} GROUP BY 1 {fav_only_join}""" else: @@ -488,8 +536,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", generic_args) - # print("--------------------") - # print(main_query) + print("--------------------") + print(main_query) cur.execute(main_query) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index b8b571d03..a9cd6afd3 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -1,6 +1,7 @@ import random import re import string +from typing import Union import math import requests @@ -168,39 +169,56 @@ def string_to_sql_like(value): def string_to_sql_like_with_op(value, op): - if isinstance(value, list) and len(value) > 0: - _value = value[0] + if isinstance(value, list): + r = [] + for v in value: + r.append(string_to_sql_like_with_op(v, op)) + return r else: _value = value - if _value is None: - return _value - if op.upper() != 'ILIKE': + if _value is None: + return _value + if op.upper() != 'ILIKE': + return _value.replace("%", "%%") + _value = _value.replace("*", "%") + if _value.startswith("^"): + _value = _value[1:] + elif not _value.startswith("%"): + _value = '%' + _value + + if _value.endswith("$"): + _value = _value[:-1] + elif not _value.endswith("%"): + _value = _value + '%' return _value.replace("%", "%%") - _value = _value.replace("*", "%") - if _value.startswith("^"): - _value = _value[1:] - elif not _value.startswith("%"): - _value = '%' + _value - - if _value.endswith("$"): - _value = _value[:-1] - elif not _value.endswith("%"): - _value = _value + '%' - return _value.replace("%", "%%") -def string_to_op(value: str, op: schemas.SearchEventOperator): - if isinstance(value, list) and len(value) > 0: - _value = value[0] +likable_operators = [schemas.SearchEventOperator._starts_with, schemas.SearchEventOperator._ends_with, + schemas.SearchEventOperator._contains, schemas.SearchEventOperator._notcontains] + + +def is_likable(op: schemas.SearchEventOperator): + return op in likable_operators + + +def values_for_operator(value: Union[str, list], op: schemas.SearchEventOperator): + if not is_likable(op): + return value + if isinstance(value, list): + r = [] + for v in value: + r.append(values_for_operator(v, op)) + return r else: - _value = value - if _value is None: - return _value - if op == schemas.SearchEventOperator._starts_with: - _value = '^' + _value - elif op == schemas.SearchEventOperator._ends_with: - _value = _value + '$' - return _value + if value is None: + return value + if op == schemas.SearchEventOperator._starts_with: + return value + '%%' + elif op == schemas.SearchEventOperator._ends_with: + return '%%' + value + elif op == schemas.SearchEventOperator._contains: + return '%%' + value + '%%' + return value def is_valid_email(email): diff --git a/api/schemas.py b/api/schemas.py index 5ec835dd1..b432dd0ac 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -330,6 +330,36 @@ class SourcemapUploadPayloadSchema(BaseModel): urls: List[str] = Field(..., alias="URL") +class ErrorSource(str, Enum): + js_exception = "js_exception" + bugsnag = "bugsnag" + cloudwatch = "cloudwatch" + datadog = "datadog" + newrelic = "newrelic" + rollbar = "rollbar" + sentry = "sentry" + stackdriver = "stackdriver" + sumologic = "sumologic" + + +class EventType(str, Enum): + click = "CLICK" + input = "INPUT" + location = "LOCATION" + custom = "CUSTOM" + request = "REQUEST" + graphql = "GRAPHQL" + state_action = "STATEACTION" + error = "ERROR" + metadata = "METADATA" + click_ios = "CLICK_IOS" + input_ios = "INPUT_IOS" + view_ios = "VIEW_IOS" + custom_ios = "CUSTOM_IOS" + request_ios = "REQUEST_IOS" + error_ios = "ERROR_IOS" + + class SearchEventOperator(str, Enum): _is = "is" _is_any = "isAny" @@ -348,13 +378,19 @@ class PlatformType(str, Enum): desktop = "desktop" +class SearchEventOrder(str, Enum): + _then = "then" + _or = "or" + _and = "and" + + class _SessionSearchEventSchema(BaseModel): - custom: Optional[str] = Field(None) + custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) value: Union[Optional[str], Optional[List[str]]] = Field(...) - type: str = Field(...) + type: EventType = Field(...) operator: SearchEventOperator = Field(...) - source: Optional[str] = Field(...) + source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) class _SessionSearchFilterSchema(_SessionSearchEventSchema): @@ -371,6 +407,10 @@ class SessionsSearchPayloadSchema(BaseModel): sort: str = Field(...) order: str = Field(default="DESC") platform: Optional[PlatformType] = Field(None) + events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then) + + class Config: + alias_generator = attribute_to_camel_case class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): From 7ac3e20b5f18d4865c964e053801f905171c2eb0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Dec 2021 12:08:56 +0100 Subject: [PATCH 002/206] feat(api): fixed search sessions or condition --- api/chalicelib/core/sessions.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index ee732c848..62920a02a 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -272,7 +272,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints = [s.decode('UTF-8') for s in ss_constraints] events_query_from = [] event_index = 0 - events_joiner = " FULL JOIN " if data.events_order == schemas.SearchEventOrder._or else " INNER JOIN LATERAL " + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " + events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " for event in data.events: event_type = event.type.upper() if not isinstance(event.value, list): @@ -282,7 +284,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f if __is_negation_operator(event.operator): is_not = True op = __reverse_sql_operator(op) - if event_index == 0 or data.events_order == schemas.SearchEventOrder._or: + if event_index == 0 or or_events: event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", @@ -401,7 +403,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: continue - if event_index == 0 or data.events_order == schemas.SearchEventOrder._or: + if event_index == 0 or or_events: event_where += ss_constraints if is_not: if event_index == 0: @@ -434,13 +436,22 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY 1 - ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ """, {**generic_args, **event_args}).decode('UTF-8')) event_index += 1 if event_index > 0: - events_query_part = f"""SELECT - event_0.session_id, - MIN(event_0.timestamp) AS first_event_ts, + if or_events: + events_query_part = f"""SELECT + session_id, + MIN(timestamp) AS first_event_ts, + MAX(timestamp) AS last_event_ts + FROM ({events_joiner.join(events_query_from)}) AS u + GROUP BY 1 + {fav_only_join}""" + else: + events_query_part = f"""SELECT + event_0.session_id, + MIN(event_0.timestamp) AS first_event_ts, MAX(event_{event_index - 1}.timestamp) AS last_event_ts FROM {events_joiner.join(events_query_from)} GROUP BY 1 From fd920d38341048ddaabaf2c57544c0e2e6e0ac9f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Dec 2021 12:34:19 +0100 Subject: [PATCH 003/206] feat(api): search sessions support isAny and onAny event-operator --- api/chalicelib/core/sessions.py | 83 +++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 62920a02a..f80c363ab 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -276,7 +276,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " for event in data.events: - event_type = event.type.upper() + event_type = event.type + is_any = event.operator in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] if not isinstance(event.value, list): event.value = [event.value] op = __get_sql_operator(event.operator) @@ -305,14 +306,18 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f continue if event_type == events.event_type.CLICK.ui_type: event_from = event_from % f"{events.event_type.CLICK.table} AS main " - event_where.append(__multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") elif event_type == events.event_type.INPUT.ui_type: event_from = event_from % f"{events.event_type.INPUT.table} AS main " - event_where.append(__multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") if len(event.custom) > 0: event_where.append(__multiple_conditions(f"main.value ILIKE %(custom)s", @@ -322,36 +327,46 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - event_where.append(__multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") elif event_type == events.event_type.CUSTOM.ui_type: event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " - event_where.append(__multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") elif event_type == events.event_type.REQUEST.ui_type: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - event_where.append(__multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") elif event_type == events.event_type.GRAPHQL.ui_type: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " - event_where.append(__multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") elif event_type == events.event_type.STATEACTION.ui_type: event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " - event_where.append( - __multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") elif event_type == events.event_type.ERROR.ui_type: # if event.source in [None, "*", ""]: # event.source = "js_exception" event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" if event.value not in [None, "*", ""]: - event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)") + if not is_any: + event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)") if event.source not in [None, "*", ""]: event_where.append(f"main1.source = %(source)s") event_args["source"] = event.source @@ -362,14 +377,18 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ----- IOS elif event_type == events.event_type.CLICK_IOS.ui_type: event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " - event_where.append( - __multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s", event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") elif event_type == events.event_type.INPUT_IOS.ui_type: event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " - event_where.append( - __multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s", event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") if len(event.custom) > 0: event_where.append(__multiple_conditions("main.value ILIKE %(custom)s", event.custom)) @@ -378,24 +397,28 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " - event_where.append( - __multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s", event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") elif event_type == events.event_type.CUSTOM_IOS.ui_type: event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " - event_where.append( - __multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") elif event_type == events.event_type.REQUEST_IOS.ui_type: event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " - event_where.append( - __multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s", - event.value)) + if not is_any: + event_where.append( + __multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s", + event.value)) # event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") elif event_type == events.event_type.ERROR_IOS.ui_type: event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" - if event.value not in [None, "*", ""]: + if not is_any and event.value not in [None, "*", ""]: event_where.append( __multiple_conditions(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)", event.value)) From 17773e126fc37a0342c10241363e7cf4d327494c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Dec 2021 15:09:20 +0100 Subject: [PATCH 004/206] feat(api): search sessions multi-value-filters feat(api): search sessions support device_type --- api/chalicelib/core/sessions.py | 159 ++++++++++++++++---------- api/chalicelib/core/sessions_metas.py | 29 ++--- api/chalicelib/utils/helper.py | 2 +- api/schemas.py | 37 +++++- 4 files changed, 147 insertions(+), 80 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index f80c363ab..8e6bcf853 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -104,29 +104,25 @@ def get_by_id2_pg(project_id, session_id, user_id, full_data=False, include_fav_ return None -def __is_multivalue(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._is_any, schemas.SearchEventOperator._on_any] - - def __get_sql_operator(op: schemas.SearchEventOperator): return { schemas.SearchEventOperator._is: "=", schemas.SearchEventOperator._is_any: "IN", schemas.SearchEventOperator._on: "=", schemas.SearchEventOperator._on_any: "IN", - schemas.SearchEventOperator._isnot: "!=", - schemas.SearchEventOperator._noton: "!=", + schemas.SearchEventOperator._is_not: "!=", + schemas.SearchEventOperator._not_on: "!=", schemas.SearchEventOperator._contains: "ILIKE", - schemas.SearchEventOperator._notcontains: "NOT ILIKE", + schemas.SearchEventOperator._not_contains: "NOT ILIKE", schemas.SearchEventOperator._starts_with: "ILIKE", schemas.SearchEventOperator._ends_with: "ILIKE", }.get(op, "=") def __is_negation_operator(op: schemas.SearchEventOperator): - return op in [schemas.SearchEventOperator._isnot, - schemas.SearchEventOperator._noton, - schemas.SearchEventOperator._notcontains] + return op in [schemas.SearchEventOperator._is_not, + schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] def __reverse_sql_operator(op): @@ -134,8 +130,8 @@ def __reverse_sql_operator(op): def __get_sql_operator_multiple(op: schemas.SearchEventOperator): - # op == schemas.SearchEventOperator._is is for filter support - return " IN " if __is_multivalue(op) or op == schemas.SearchEventOperator._is else " NOT IN " + return " IN " if op not in [schemas.SearchEventOperator._is_not, schemas.SearchEventOperator._not_on, + schemas.SearchEventOperator._not_contains] else " NOT IN " def __get_sql_value_multiple(values): @@ -144,12 +140,12 @@ def __get_sql_value_multiple(values): return tuple(values) if isinstance(values, list) else (values,) -def __multiple_conditions(condition, values, value_key="value"): +def __multiple_conditions(condition, values, value_key="value", is_not=False): query = [] for i in range(len(values)): k = f"{value_key}_{i}" query.append(condition.replace("value", k)) - return "(" + " OR ".join(query) + ")" + return "(" + (" AND " if is_not else " OR ").join(query) + ")" def __multiple_values(values, value_key="value"): @@ -181,41 +177,63 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f events_query_part = "" if len(data.filters) > 0: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} + meta_keys = None for f in data.filters: if not isinstance(f.value, list): f.value = [f.value] if len(f.value) == 0 or f.value[0] is None: continue - filter_type = f.type.upper() - f.value = __get_sql_value_multiple(f.value) + filter_type = f.type + # f.value = __get_sql_value_multiple(f.value) + f.value = helper.values_for_operator(value=f.value, op=f.operator) + filter_args = __multiple_values(f.value) + op = __get_sql_operator(f.operator) + is_not = False + if __is_negation_operator(f.operator): + is_not = True + # op = __reverse_sql_operator(op) if filter_type == sessions_metas.meta_type.USERBROWSER: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_browser {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_browser {op} %(value)s', {"value": f.value})) + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_browser {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_browser {op} %(value)s', f.value, is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_os {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_os {op} %(value)s', {"value": f.value})) + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_os {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_os {op} %(value)s', f.value, is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_device {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_device {op} %(value)s', {"value": f.value})) + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_device {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_device {op} %(value)s', f.value, is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: - op = __get_sql_operator_multiple(f.operator) - extra_constraints.append(cur.mogrify(f's.user_country {op} %(value)s', {"value": f.value})) - ss_constraints.append(cur.mogrify(f'ms.user_country {op} %(value)s', {"value": f.value})) - elif filter_type == "duration".upper(): + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f's.user_country {op} %(value)s', f.value, is_not=is_not), + filter_args)) + ss_constraints.append( + cur.mogrify(__multiple_conditions(f'ms.user_country {op} %(value)s', f.value, is_not=is_not), + filter_args)) + elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: extra_constraints.append( cur.mogrify("s.duration >= %(minDuration)s", {"minDuration": f.value[0]})) ss_constraints.append( cur.mogrify("ms.duration >= %(minDuration)s", {"minDuration": f.value[0]})) - if len(f.value) > 1 and f.value[1] is not None and f.value[1] > 0: + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: extra_constraints.append( cur.mogrify("s.duration <= %(maxDuration)s", {"maxDuration": f.value[1]})) ss_constraints.append( @@ -223,48 +241,69 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f elif filter_type == sessions_metas.meta_type.REFERRER: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - op = __get_sql_operator_multiple(f.operator) + # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(f"p.base_referrer {op} %(referrer)s", {"referrer": f.value})) + cur.mogrify(__multiple_conditions(f"p.base_referrer {op} %(value)s", f.value, is_not=is_not), + filter_args)) elif filter_type == events.event_type.METADATA.ui_type: - op = __get_sql_operator(f.operator) + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + # op = __get_sql_operator(f.operator) if f.key in meta_keys.keys(): extra_constraints.append( - cur.mogrify(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)})) + cur.mogrify( + __multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", + f.value, is_not=is_not), filter_args)) ss_constraints.append( - cur.mogrify(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)})) + cur.mogrify(__multiple_conditions( + f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", f.value, + is_not=is_not), + filter_args)) elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: - op = __get_sql_operator(f.operator) + # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(f"s.user_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"s.user_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) ss_constraints.append( - cur.mogrify(f"ms.user_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"ms.user_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS]: - op = __get_sql_operator(f.operator) + # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(f"s.user_anonymous_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify( + __multiple_conditions(f"s.user_anonymous_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) ss_constraints.append( - cur.mogrify(f"ms.user_anonymous_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify( + __multiple_conditions(f"ms.user_anonymous_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: - op = __get_sql_operator(f.operator) + # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(f"s.rev_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"s.rev_id {op} %(value)s", f.value, is_not=is_not), + filter_args) ) ss_constraints.append( - cur.mogrify(f"ms.rev_id {op} %(value)s", - {"value": helper.string_to_sql_like_with_op(f.value[0], op)}) + cur.mogrify(__multiple_conditions(f"ms.rev_id {op} %(value)s", f.value, is_not=is_not), + filter_args) + ) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + cur.mogrify(__multiple_conditions(f"s.user_device_type {op} %(value)s", f.value, is_not=is_not), + filter_args) + ) + ss_constraints.append( + cur.mogrify( + __multiple_conditions(f"ms.user_device_type {op} %(value)s", f.value, is_not=is_not), + filter_args) ) # --------------------------------------------------------------------------- @@ -493,12 +532,12 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: data.endDate = None - if data.platform is not None: - if data.platform == schemas.PlatformType.mobile: - extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") - elif data.platform == schemas.PlatformType.desktop: - extra_constraints.append( - b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") order = "DESC" if data.order is not None: diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py index a21b78783..1e55720a9 100644 --- a/api/chalicelib/core/sessions_metas.py +++ b/api/chalicelib/core/sessions_metas.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.utils import pg_client, helper from chalicelib.utils.event_filter_definition import SupportedFilter @@ -109,21 +110,21 @@ def __generic_autocomplete(typename): class meta_type: - USEROS = "USEROS" - USERBROWSER = "USERBROWSER" - USERDEVICE = "USERDEVICE" - USERCOUNTRY = "USERCOUNTRY" - USERID = "USERID" - USERANONYMOUSID = "USERANONYMOUSID" - REFERRER = "REFERRER" - REVID = "REVID" + USEROS = schemas.FilterType.user_os + USERBROWSER = schemas.FilterType.user_browser + USERDEVICE = schemas.FilterType.user_device + USERCOUNTRY = schemas.FilterType.user_country + USERID = schemas.FilterType.user_id + USERANONYMOUSID = schemas.FilterType.user_anonymous_id + REFERRER = schemas.FilterType.referrer + REVID = schemas.FilterType.rev_id # IOS - USEROS_IOS = "USEROS_IOS" - USERDEVICE_IOS = "USERDEVICE_IOS" - USERCOUNTRY_IOS = "USERCOUNTRY_IOS" - USERID_IOS = "USERID_IOS" - USERANONYMOUSID_IOS = "USERANONYMOUSID_IOS" - REVID_IOS = "REVID_IOS" + USEROS_IOS = schemas.FilterType.user_os_ios + USERDEVICE_IOS = schemas.FilterType.user_device_ios + USERCOUNTRY_IOS = schemas.FilterType.user_country_ios + USERID_IOS = schemas.FilterType.user_id_ios + USERANONYMOUSID_IOS = schemas.FilterType.user_anonymous_id_ios + REVID_IOS = schemas.FilterType.rev_id_ios SUPPORTED_TYPES = { diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index a9cd6afd3..868de0411 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -194,7 +194,7 @@ def string_to_sql_like_with_op(value, op): likable_operators = [schemas.SearchEventOperator._starts_with, schemas.SearchEventOperator._ends_with, - schemas.SearchEventOperator._contains, schemas.SearchEventOperator._notcontains] + schemas.SearchEventOperator._contains, schemas.SearchEventOperator._not_contains] def is_likable(op: schemas.SearchEventOperator): diff --git a/api/schemas.py b/api/schemas.py index b432dd0ac..69563b047 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -360,15 +360,37 @@ class EventType(str, Enum): error_ios = "ERROR_IOS" +class FilterType(str, Enum): + user_os = "USEROS" + user_browser = "USERBROWSER" + user_device = "USERDEVICE" + user_country = "USERCOUNTRY" + user_id = "USERID" + user_anonymous_id = "USERANONYMOUSID" + referrer = "REFERRER" + rev_id = "REVID" + # IOS + user_os_ios = "USEROS_IOS" + user_device_ios = "USERDEVICE_IOS" + user_country_ios = "USERCOUNTRY_IOS" + user_id_ios = "USERID_IOS" + user_anonymous_id_ios = "USERANONYMOUSID_IOS" + rev_id_ios = "REVID_IOS" + # + duration = "DURATION" + platform = "PLATFORM" + metadata = "METADATA" + + class SearchEventOperator(str, Enum): _is = "is" _is_any = "isAny" _on = "on" _on_any = "onAny" - _isnot = "isNot" - _noton = "notOn" + _is_not = "isNot" + _not_on = "notOn" _contains = "contains" - _notcontains = "notContains" + _not_contains = "notContains" _starts_with = "startsWith" _ends_with = "endsWith" @@ -393,8 +415,13 @@ class _SessionSearchEventSchema(BaseModel): source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) -class _SessionSearchFilterSchema(_SessionSearchEventSchema): - value: List[str] = Field(...) +class _SessionSearchFilterSchema(BaseModel): + custom: Optional[List[str]] = Field(None) + key: Optional[str] = Field(None) + value: Union[Optional[Union[str, int]], Optional[List[Union[str, int]]]] = Field(...) + type: FilterType = Field(...) + operator: SearchEventOperator = Field(...) + source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) class SessionsSearchPayloadSchema(BaseModel): From e02afe51da313e915124967a0cc920e7adfcd5ab Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Dec 2021 16:34:05 +0100 Subject: [PATCH 005/206] feat(api): changed build script --- api/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/build.sh b/api/build.sh index d66a54ab9..ee7cec3ab 100644 --- a/api/build.sh +++ b/api/build.sh @@ -26,11 +26,11 @@ function build_api(){ envarg="default-ee" tag="ee-" } - docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/chalice:${git_sha1} . + docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/api:${git_sha1} . [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/chalice:${git_sha1} - docker tag ${DOCKER_REPO:-'local'}/chalice:${git_sha1} ${DOCKER_REPO:-'local'}/chalice:${tag}latest - docker push ${DOCKER_REPO:-'local'}/chalice:${tag}latest + docker push ${DOCKER_REPO:-'local'}/api:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/api:${git_sha1} ${DOCKER_REPO:-'local'}/api:${tag}latest + docker push ${DOCKER_REPO:-'local'}/api:${tag}latest } } From de1d9d08f7793f5b0ee421c364fe0349bf3d0306 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 21 Dec 2021 19:15:36 +0100 Subject: [PATCH 006/206] feat(api): changed build script feat(api): changed search payload schema --- api/build.sh | 2 +- api/schemas.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/build.sh b/api/build.sh index ee7cec3ab..06cc4e309 100644 --- a/api/build.sh +++ b/api/build.sh @@ -30,7 +30,7 @@ function build_api(){ [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/api:${git_sha1} docker tag ${DOCKER_REPO:-'local'}/api:${git_sha1} ${DOCKER_REPO:-'local'}/api:${tag}latest - docker push ${DOCKER_REPO:-'local'}/api:${tag}latest + docker push ${DOCKER_REPO:-'local'}/api:${tag}api } } diff --git a/api/schemas.py b/api/schemas.py index 69563b047..8fd2ae2d0 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -433,7 +433,7 @@ class SessionsSearchPayloadSchema(BaseModel): endDate: int = Field(None) sort: str = Field(...) order: str = Field(default="DESC") - platform: Optional[PlatformType] = Field(None) + # platform: Optional[PlatformType] = Field(None) events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then) class Config: From 7b795767963c7fb3a517138edbce61c0d5d3c714 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 22 Dec 2021 20:29:37 +0100 Subject: [PATCH 007/206] feat(api): changed sessions-search query build logic feat(api): funnel support new operators --- api/chalicelib/core/sessions.py | 246 ++++++++++++---------------- api/chalicelib/core/significance.py | 126 ++++++++------ api/chalicelib/utils/helper.py | 6 +- api/schemas.py | 3 + 4 files changed, 185 insertions(+), 196 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8e6bcf853..caa19db1e 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -140,15 +140,15 @@ def __get_sql_value_multiple(values): return tuple(values) if isinstance(values, list) else (values,) -def __multiple_conditions(condition, values, value_key="value", is_not=False): +def _multiple_conditions(condition, values, value_key="value", is_not=False): query = [] for i in range(len(values)): k = f"{value_key}_{i}" - query.append(condition.replace("value", k)) + query.append(condition.replace(value_key, k)) return "(" + (" AND " if is_not else " OR ").join(query) + ")" -def __multiple_values(values, value_key="value"): +def _multiple_values(values, value_key="value"): query_values = {} for i in range(len(values)): k = f"{value_key}_{i}" @@ -156,29 +156,33 @@ def __multiple_values(values, value_key="value"): return query_values +def _isAny_opreator(op: schemas.SearchEventOperator): + return op in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] + + @dev.timed def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, favorite_only=False, errors_only=False, error_status="ALL", count_only=False, issue=None): - generic_args = {"startDate": data.startDate, "endDate": data.endDate, - "projectId": project_id, - "userId": user_id} with pg_client.PostgresClient() as cur: ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} extra_constraints = [ - cur.mogrify("s.project_id = %(project_id)s", {"project_id": project_id}), - cur.mogrify("s.duration IS NOT NULL", {}) + "s.project_id = %(project_id)s", + "s.duration IS NOT NULL" ] extra_from = "" fav_only_join = "" if favorite_only and not errors_only: fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" - extra_constraints.append(cur.mogrify("fs.user_id = %(userId)s", {"userId": user_id})) + extra_constraints.append("fs.user_id = %(userId)s") + full_args["userId"] = user_id events_query_part = "" if len(data.filters) > 0: meta_keys = None - for f in data.filters: + for i, f in enumerate(data.filters): if not isinstance(f.value, list): f.value = [f.value] if len(f.value) == 0 or f.value[0] is None: @@ -186,7 +190,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f filter_type = f.type # f.value = __get_sql_value_multiple(f.value) f.value = helper.values_for_operator(value=f.value, op=f.operator) - filter_args = __multiple_values(f.value) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} op = __get_sql_operator(f.operator) is_not = False if __is_negation_operator(f.operator): @@ -195,56 +200,45 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f if filter_type == sessions_metas.meta_type.USERBROWSER: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f's.user_browser {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions(f'ms.user_browser {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f's.user_os {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions(f'ms.user_os {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f's.user_device {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions(f'ms.user_device {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f's.user_country {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions(f'ms.user_country {op} %(value)s', f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: - extra_constraints.append( - cur.mogrify("s.duration >= %(minDuration)s", {"minDuration": f.value[0]})) - ss_constraints.append( - cur.mogrify("ms.duration >= %(minDuration)s", {"minDuration": f.value[0]})) + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: - extra_constraints.append( - cur.mogrify("s.duration <= %(maxDuration)s", {"maxDuration": f.value[1]})) - ss_constraints.append( - cur.mogrify("ms.duration <= %(maxDuration)s", {"maxDuration": f.value[1]})) + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] elif filter_type == sessions_metas.meta_type.REFERRER: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f"p.base_referrer {op} %(value)s", f.value, is_not=is_not), - filter_args)) + _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) elif filter_type == events.event_type.METADATA.ui_type: # get metadata list only if you need it if meta_keys is None: @@ -253,70 +247,52 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # op = __get_sql_operator(f.operator) if f.key in meta_keys.keys(): extra_constraints.append( - cur.mogrify( - __multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", - f.value, is_not=is_not), filter_args)) + _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions( - f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %(value)s", f.value, - is_not=is_not), - filter_args)) + _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + f.value, is_not=is_not, value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f"s.user_id {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions(f"ms.user_id {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS]: # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify( - __multiple_conditions(f"s.user_anonymous_id {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - cur.mogrify( - __multiple_conditions(f"ms.user_anonymous_id {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f"s.rev_id {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - cur.mogrify(__multiple_conditions(f"ms.rev_id {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"ms.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) elif filter_type == schemas.FilterType.platform: # op = __get_sql_operator(f.operator) extra_constraints.append( - cur.mogrify(__multiple_conditions(f"s.user_device_type {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) ss_constraints.append( - cur.mogrify( - __multiple_conditions(f"ms.user_device_type {op} %(value)s", f.value, is_not=is_not), - filter_args) - ) + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: - ss_constraints = [s.decode('UTF-8') for s in ss_constraints] + # ss_constraints = [s.decode('UTF-8') for s in ss_constraints] events_query_from = [] event_index = 0 or_events = data.events_order == schemas.SearchEventOrder._or # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " - for event in data.events: + for i, event in enumerate(data.events): event_type = event.type - is_any = event.operator in [schemas.SearchEventOperator._on_any, schemas.SearchEventOperator._is_any] + is_any = _isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] op = __get_sql_operator(event.operator) @@ -337,7 +313,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") event.value = helper.values_for_operator(value=event.value, op=event.operator) - event_args = __multiple_values(event.value) + # event_args = _multiple_values(event.value) + e_k = f"e_value{i}" + full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} if event_type not in list(events.SUPPORTED_TYPES.keys()) \ or event.value in [None, "", "*"] \ and (event_type != events.event_type.ERROR.ui_type \ @@ -347,121 +325,106 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_from = event_from % f"{events.event_type.CLICK.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.CLICK.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.INPUT.ui_type: event_from = event_from % f"{events.event_type.INPUT.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.INPUT.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) if len(event.custom) > 0: - event_where.append(__multiple_conditions(f"main.value ILIKE %(custom)s", - event.custom, value_key="custom")) - event_args = {**event_args, **__multiple_values(event.custom, value_key="custom")} - # event_where.append("main.value ILIKE %(custom)s") - # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} + elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.LOCATION.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.CUSTOM.ui_type: event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.CUSTOM.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.REQUEST.ui_type: event_from = event_from % f"{events.event_type.REQUEST.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.REQUEST.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.GRAPHQL.ui_type: event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.GRAPHQL.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) elif event_type == events.event_type.STATEACTION.ui_type: event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.STATEACTION.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.ERROR.ui_type: # if event.source in [None, "*", ""]: # event.source = "js_exception" event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" if event.value not in [None, "*", ""]: if not is_any: - event_where.append(f"(main1.message {op} %(value)s OR main1.name {op} %(value)s)") + event_where.append(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)") if event.source not in [None, "*", ""]: event_where.append(f"main1.source = %(source)s") - event_args["source"] = event.source + full_args["source"] = event.source elif event.source not in [None, "*", ""]: event_where.append(f"main1.source = %(source)s") - event_args["source"] = event.source + full_args["source"] = event.source # ----- IOS elif event_type == events.event_type.CLICK_IOS.ui_type: event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.CLICK_IOS.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.INPUT_IOS.ui_type: event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.INPUT_IOS.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) if len(event.custom) > 0: - event_where.append(__multiple_conditions("main.value ILIKE %(custom)s", event.custom)) - event_args = {**event_args, **__multiple_values(event.custom, "custom")} - # event_where.append("main.value ILIKE %(custom)s") - # event_args["custom"] = helper.string_to_sql_like_with_op(event.custom, "ILIKE") + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + value_key="custom{i}")) + full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.VIEW_IOS.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.CUSTOM_IOS.ui_type: event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.CUSTOM_IOS.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.REQUEST_IOS.ui_type: event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " if not is_any: event_where.append( - __multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s", - event.value)) - # event_where.append(f"main.{events.event_type.REQUEST_IOS.column} {op} %(value)s") + _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) elif event_type == events.event_type.ERROR_IOS.ui_type: event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" if not is_any and event.value not in [None, "*", ""]: event_where.append( - __multiple_conditions(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)", - event.value)) - # event_where.append(f"(main1.reason {op} %(value)s OR main1.name {op} %(value)s)") + _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) else: continue @@ -469,7 +432,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where += ss_constraints if is_not: if event_index == 0: - events_query_from.append(cur.mogrify(f"""\ + events_query_from.append(f"""\ (SELECT session_id, 0 AS timestamp @@ -483,23 +446,23 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f AND start_ts <= %(endDate)s AND duration IS NOT NULL ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ - """, {**generic_args, **event_args}).decode('UTF-8')) + """) else: - events_query_from.append(cur.mogrify(f"""\ + events_query_from.append(f"""\ (SELECT event_0.session_id, event_{event_index - 1}.timestamp AS timestamp WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ - """, {**generic_args, **event_args}).decode('UTF-8')) + """) else: - events_query_from.append(cur.mogrify(f"""\ + events_query_from.append(f"""\ (SELECT main.session_id, MIN(timestamp) AS timestamp FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY 1 - ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ - """, {**generic_args, **event_args}).decode('UTF-8')) + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) event_index += 1 if event_index > 0: if or_events: @@ -524,13 +487,10 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # --------------------------------------------------------------------------- if data.startDate is not None: - extra_constraints.append(cur.mogrify("s.start_ts >= %(startDate)s", {"startDate": data.startDate})) - else: - data.startDate = None + extra_constraints.append("s.start_ts >= %(startDate)s") + if data.endDate is not None: - extra_constraints.append(cur.mogrify("s.start_ts <= %(endDate)s", {"endDate": data.endDate})) - else: - data.endDate = None + extra_constraints.append("s.start_ts <= %(endDate)s") # if data.platform is not None: # if data.platform == schemas.PlatformType.mobile: @@ -557,7 +517,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" extra_constraints.append(cur.mogrify("ufe.user_id = %(user_id)s", {"user_id": user_id})) - extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] if not favorite_only and not errors_only: extra_from += """LEFT JOIN (SELECT user_id, session_id FROM public.user_favorite_sessions @@ -594,20 +554,20 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f WHERE er.error_id = ve.error_id AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed {query_part};""", - generic_args) + full_args) elif count_only: main_query = cur.mogrify( f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, COUNT(DISTINCT s.user_uuid) AS count_users {query_part};""", - generic_args) + full_args) else: main_query = cur.mogrify(f"""SELECT * FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {query_part} ORDER BY s.session_id desc) AS filtred_sessions ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", - generic_args) + full_args) print("--------------------") print(main_query) diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 8bcda04a7..3f991b895 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -1,6 +1,7 @@ __author__ = "AZNAUROV David" __maintainer__ = "KRAIEM Taha Yassine" +import schemas from chalicelib.core import events, sessions_metas, metadata, sessions from chalicelib.utils import dev @@ -30,87 +31,107 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: :param filter_d: dict contains events&filters&... :return: """ - stages = filter_d["events"] - filters = filter_d.get("filters", []) + stages: [dict] = filter_d["events"] + filters: [dict] = filter_d.get("filters", []) filter_issues = filter_d.get("issueTypes") if filter_issues is None or len(filter_issues) == 0: filter_issues = [] stage_constraints = ["main.timestamp <= %(endTimestamp)s"] first_stage_extra_constraints = ["s.project_id=%(project_id)s", "s.start_ts >= %(startTimestamp)s", "s.start_ts <= %(endTimestamp)s"] - extra_from = "" + filter_extra_from = [] n_stages_query = [] values = {} if len(filters) > 0: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} + meta_keys = None for i, f in enumerate(filters): - if not isinstance(f.get("value"), list): - if isinstance(f.get("value"), tuple): - f["value"] = list(f.get("value")) - else: - f["value"] = [f.get("value")] - if len(f["value"]) == 0 or f["value"][0] is None: + if not isinstance(f["value"], list): + f.value = [f["value"]] + if len(f["value"]) == 0 or f["value"] is None: continue - filter_type = f["type"].upper() - values[f"f_value_{i}"] = sessions.__get_sql_value_multiple(f["value"]) + f["value"] = helper.values_for_operator(value=f["value"], op=f["operator"]) + # filter_args = _multiple_values(f["value"]) + op = sessions.__get_sql_operator(f["operator"]) + + filter_type = f["type"] + # values[f_k] = sessions.__get_sql_value_multiple(f["value"]) + f_k = f"f_value{i}" + values = {**values, + **sessions._multiple_values(helper.values_for_operator(value=f["value"], op=f["operator"]), + value_key=f_k)} if filter_type == sessions_metas.meta_type.USERBROWSER: - op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append(f's.user_browser {op} %({f"f_value_{i}"})s') + # op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: - op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append(f's.user_os {op} %({f"f_value_{i}"})s') + # op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: - op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append(f's.user_device {op} %({f"f_value_{i}"})s') + # op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k)) elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: - op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append(f's.user_country {op} %({f"f_value_{i}"})s') - elif filter_type == "duration".upper(): + # op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k)) + elif filter_type == schemas.FilterType.duration: if len(f["value"]) > 0 and f["value"][0] is not None: - first_stage_extra_constraints.append(f's.duration >= %({f"f_value_{i}"})s') - values[f"f_value_{i}"] = f["value"][0] - if len(f["value"]) > 1 and f["value"][1] is not None and f["value"][1] > 0: - first_stage_extra_constraints.append('s.duration <= %({f"f_value_{i}"})s') - values[f"f_value_{i}"] = f["value"][1] + first_stage_extra_constraints.append(f's.duration >= %(minDuration)s') + values["minDuration"] = f["value"][0] + if len(f["value"]) > 1 and f["value"][1] is not None and int(f["value"][1]) > 0: + first_stage_extra_constraints.append('s.duration <= %(maxDuration)s') + values["maxDuration"] = f["value"][1] elif filter_type == sessions_metas.meta_type.REFERRER: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" - extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - op = sessions.__get_sql_operator_multiple(f["operator"]) - first_stage_extra_constraints.append(f"p.base_referrer {op} %(referrer)s") + filter_extra_from = [f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)"] + # op = sessions.__get_sql_operator_multiple(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f"p.base_referrer {op} %({f_k})s", f["value"], value_key=f_k)) elif filter_type == events.event_type.METADATA.ui_type: - op = sessions.__get_sql_operator(f["operator"]) + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + # op = sessions.__get_sql_operator(f["operator"]) if f.get("key") in meta_keys.keys(): first_stage_extra_constraints.append( - f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f"f_value_{i}"})s') - values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + sessions._multiple_conditions( + f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f_k})s', f["value"], + value_key=f_k)) + # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: - op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append(f's.user_id {op} %({f"f_value_{i}"})s') - values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + # op = sessions.__get_sql_operator(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k)) + # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS]: - op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append(f's.user_anonymous_id {op} %({f"f_value_{i}"})s') - values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + # op = sessions.__get_sql_operator(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k)) + # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: - op = sessions.__get_sql_operator(f["operator"]) - first_stage_extra_constraints.append(f's.rev_id {op} %({f"f_value_{i}"})s') - values[f"f_value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) + # op = sessions.__get_sql_operator(f["operator"]) + first_stage_extra_constraints.append( + sessions._multiple_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k)) + # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) for i, s in enumerate(stages): if i == 0: - extra_from = ["INNER JOIN public.sessions AS s USING (session_id)"] + extra_from = filter_extra_from + ["INNER JOIN public.sessions AS s USING (session_id)"] else: extra_from = [] if s.get("operator") is None: s["operator"] = "is" + + if not isinstance(s["value"], list): + s["value"] = [s["value"]] + is_any = sessions._isAny_opreator(s["operator"]) op = sessions.__get_sql_operator(s["operator"]) event_type = s["type"].upper() - next_label = s["value"] if event_type == events.event_type.CLICK.ui_type: next_table = events.event_type.CLICK.table next_col_name = events.event_type.CLICK.column @@ -140,7 +161,8 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: print("=================UNDEFINED") continue - values[f"value{i + 1}"] = helper.string_to_sql_like_with_op(next_label, op) + values = {**values, **sessions._multiple_values(helper.values_for_operator(value=s["value"], op=s["operator"]), + value_key=f"value{i + 1}")} if sessions.__is_negation_operator(op) and i > 0: op = sessions.__reverse_sql_operator(op) main_condition = "left_not.session_id ISNULL" @@ -150,7 +172,11 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: AND s_main.timestamp >= T{i}.stage{i}_timestamp AND s_main.session_id = T1.session_id) AS left_not ON (TRUE)""") else: - main_condition = f"""main.{next_col_name} {op} %(value{i + 1})s""" + if is_any: + main_condition = "TRUE" + else: + main_condition = sessions._multiple_conditions(f"main.{next_col_name} {op} %(value{i + 1})s", + values=s["value"], value_key=f"value{i + 1}") n_stages_query.append(f""" (SELECT main.session_id, {"MIN(main.timestamp)" if i + 1 < len(stages) else "MAX(main.timestamp)"} AS stage{i + 1}_timestamp, @@ -197,9 +223,9 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"], "issueTypes": tuple(filter_issues), **values} with pg_client.PostgresClient() as cur: - # print("---------------------------------------------------") - # print(cur.mogrify(n_stages_query, params)) - # print("---------------------------------------------------") + print("---------------------------------------------------") + print(cur.mogrify(n_stages_query, params)) + print("---------------------------------------------------") cur.execute(cur.mogrify(n_stages_query, params)) rows = cur.fetchall() return rows @@ -535,7 +561,7 @@ def get_top_insights(filter_d, project_id): "dropDueToIssues": 0 }] - counts = sessions.search2_pg(data=filter_d, project_id=project_id, user_id=None, count_only=True) + counts = sessions.search2_pg(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), project_id=project_id, user_id=None, count_only=True) output[0]["sessionsCount"] = counts["countSessions"] output[0]["usersCount"] = counts["countUsers"] return output, 0 diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index 868de0411..f8ce9fab5 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -213,11 +213,11 @@ def values_for_operator(value: Union[str, list], op: schemas.SearchEventOperator if value is None: return value if op == schemas.SearchEventOperator._starts_with: - return value + '%%' + return value + '%' elif op == schemas.SearchEventOperator._ends_with: - return '%%' + value + return '%' + value elif op == schemas.SearchEventOperator._contains: - return '%%' + value + '%%' + return '%' + value + '%' return value diff --git a/api/schemas.py b/api/schemas.py index 8fd2ae2d0..e0b1ace55 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -439,6 +439,9 @@ class SessionsSearchPayloadSchema(BaseModel): class Config: alias_generator = attribute_to_camel_case +class SessionsSearchCountSchema(SessionsSearchPayloadSchema): + sort: Optional[str] = Field(default=None) + order: Optional[str] = Field(default=None) class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): range_value: Optional[str] = Field(None) From 776d377ece9578fde7a45978095cba4ffebb1212 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 22 Dec 2021 20:35:20 +0100 Subject: [PATCH 008/206] feat(api): changed build script --- api/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build.sh b/api/build.sh index 06cc4e309..ee7cec3ab 100644 --- a/api/build.sh +++ b/api/build.sh @@ -30,7 +30,7 @@ function build_api(){ [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/api:${git_sha1} docker tag ${DOCKER_REPO:-'local'}/api:${git_sha1} ${DOCKER_REPO:-'local'}/api:${tag}latest - docker push ${DOCKER_REPO:-'local'}/api:${tag}api + docker push ${DOCKER_REPO:-'local'}/api:${tag}latest } } From 1ea681269bb910629f28fb2d3fcc2eee4228b2e3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 23 Dec 2021 20:15:36 +0100 Subject: [PATCH 009/206] feat(api): changed main sessions' search&count logic feat(DB): new index --- api/chalicelib/core/sessions.py | 41 ++++++++++++------- .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 2 + .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 5 +++ .../db/init_dbs/postgresql/init_schema.sql | 2 +- 4 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index caa19db1e..7a00de9bf 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -562,29 +562,40 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f {query_part};""", full_args) else: - main_query = cur.mogrify(f"""SELECT * FROM - (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} - {query_part} - ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", + main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200) AS sessions + FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn FROM + (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + {query_part} + ORDER BY s.session_id desc) AS filtred_sessions + ORDER BY favorite DESC, issue_score DESC, {sort} {order}) AS full_sessions;""", full_args) + # main_query = cur.mogrify(f"""SELECT * FROM + # (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} + # {query_part} + # ORDER BY s.session_id desc) AS filtred_sessions + # ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", + # full_args) + print("--------------------") print(main_query) cur.execute(main_query) - + print("Executed--------------------") if count_only: return helper.dict_to_camel_case(cur.fetchone()) - sessions = [] - total = cur.rowcount - row = cur.fetchone() - limit = 200 - while row is not None and len(sessions) < limit: - if row.get("favorite"): - limit += 1 - sessions.append(row) - row = cur.fetchone() + sessions = cur.fetchone() + total = sessions["count"] + sessions = sessions["sessions"] + # sessions = [] + # total = cur.rowcount + # row = cur.fetchone() + # limit = 200 + # while row is not None and len(sessions) < limit: + # if row.get("favorite"): + # limit += 1 + # sessions.append(row) + # row = cur.fetchone() if errors_only: return sessions diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql index 5f0f4f054..eac07d08e 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -15,4 +15,6 @@ CREATE TABLE traces ); CREATE INDEX traces_user_id_idx ON traces (user_id); CREATE INDEX traces_tenant_id_idx ON traces (tenant_id); + +CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions(user_id,session_id); COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql new file mode 100644 index 000000000..d6a3abf83 --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -0,0 +1,5 @@ +BEGIN; + +CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); + +END; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 80b2a9135..345bf6b2c 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -598,7 +598,7 @@ $$ session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, PRIMARY KEY (user_id, session_id) ); - + CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); -- --- assignments.sql --- From bd957f97f8fb13589c7b3e057bd4f0e2ba848241 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 24 Dec 2021 13:06:32 +0530 Subject: [PATCH 010/206] feat(ui) - fetch error nav, and highlight row, assist tabs --- .../components/AssistTabs/AssistTabs.tsx | 68 +++++ .../components/AssistTabs/assistTabs.css | 0 .../Assist/components/AssistTabs/index.ts | 1 + .../BugFinder/SessionsMenu/SessionsMenu.js | 2 +- frontend/app/components/Session/LivePlayer.js | 4 +- frontend/app/components/Session/Session.js | 11 + .../app/components/Session_/Fetch/Fetch.js | 89 +++++-- .../components/Session_/Network/Network.js | 247 ++++-------------- .../components/Session_/Network/network.css | 2 +- .../Session_/Player/Controls/Timeline.js | 22 +- .../components/Session_/PlayerBlockHeader.js | 2 + .../Session_/TimeTable/TimeTable.js | 8 +- .../Session_/TimeTable/timeTable.css | 4 +- .../app/components/Session_/autoscroll.css | 4 +- .../components/shared/SessionItem/Counter.tsx | 2 +- .../components/ui/IconButton/iconButton.css | 1 + frontend/app/duck/sessions.js | 17 +- frontend/app/svg/icons/os/fedora.svg | 5 + frontend/app/types/filter/filter.js | 4 +- 19 files changed, 246 insertions(+), 247 deletions(-) create mode 100644 frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx create mode 100644 frontend/app/components/Assist/components/AssistTabs/assistTabs.css create mode 100644 frontend/app/components/Assist/components/AssistTabs/index.ts create mode 100644 frontend/app/svg/icons/os/fedora.svg diff --git a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx new file mode 100644 index 000000000..8f0641200 --- /dev/null +++ b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx @@ -0,0 +1,68 @@ +import React, { useEffect, useState } from 'react'; +import { connect } from 'react-redux'; +import { applyFilter, addAttribute } from 'Duck/filters'; +import { fetchList } from 'Duck/sessions'; +import { KEYS } from 'Types/filter/customFilter'; +import { Link } from 'UI'; +import Filter from 'Types/filter'; +import { List } from 'immutable'; +import Counter from 'App/components/shared/SessionItem/Counter'; +import { fetchLiveList } from 'Duck/sessions'; +import { session as sessionRoute } from 'App/routes'; +import { session } from 'App/components/Session_/session.css'; + +const RowItem = ({ startedAt, sessionId }) => { + return ( + +
+ Tab1 + +
+ + ); +} + +interface Props { + list: List, + session: any, + fetchLiveList: () => void, + applyFilter: () => void, + filters: Filter + addAttribute: (obj) => void, + loading: boolean +} + +const AssistTabs = React.memo((props: Props) => { + const [showMenu, setShowMenu] = useState(false) + useEffect(() => { + if (!props.loading && props.list.size === 0) { + props.fetchLiveList(); + } + }, [props.list]) + + return ( +
+
setShowMenu(!showMenu)}>Active Tabs
+ {showMenu && ( +
+ {props.list.map((item, index) => ( + + ))} +
+ )} +
+ ); +}); + +export default connect(state => { + const session = state.getIn([ 'sessions', 'current' ]); + return { + loading: state.getIn([ 'sessions', 'loading' ]), + list: state.getIn(['sessions', 'liveSessions']).filter(i => i.userId === session.userId), + session, + filters: state.getIn([ 'filters', 'appliedFilter' ]), + } +}, { applyFilter, addAttribute, fetchLiveList })(AssistTabs); \ No newline at end of file diff --git a/frontend/app/components/Assist/components/AssistTabs/assistTabs.css b/frontend/app/components/Assist/components/AssistTabs/assistTabs.css new file mode 100644 index 000000000..e69de29bb diff --git a/frontend/app/components/Assist/components/AssistTabs/index.ts b/frontend/app/components/Assist/components/AssistTabs/index.ts new file mode 100644 index 000000000..b089b3734 --- /dev/null +++ b/frontend/app/components/Assist/components/AssistTabs/index.ts @@ -0,0 +1 @@ +export { default } from './AssistTabs'; \ No newline at end of file diff --git a/frontend/app/components/BugFinder/SessionsMenu/SessionsMenu.js b/frontend/app/components/BugFinder/SessionsMenu/SessionsMenu.js index fb89f6967..7275d9ac0 100644 --- a/frontend/app/components/BugFinder/SessionsMenu/SessionsMenu.js +++ b/frontend/app/components/BugFinder/SessionsMenu/SessionsMenu.js @@ -119,7 +119,7 @@ export default connect(state => ({ activeFlow: state.getIn([ 'filters', 'activeFlow' ]), captureRate: state.getIn(['watchdogs', 'captureRate']), filters: state.getIn([ 'filters', 'appliedFilter' ]), - sessionsLoading: state.getIn([ 'sessions', 'loading' ]), + sessionsLoading: state.getIn([ 'sessions', 'fetchLiveListRequest', 'loading' ]), }), { fetchWatchdogStatus, setActiveFlow, clearEvents, setActiveTab, fetchSessionList })(SessionsMenu); diff --git a/frontend/app/components/Session/LivePlayer.js b/frontend/app/components/Session/LivePlayer.js index cbc3410cd..07b8d3adc 100644 --- a/frontend/app/components/Session/LivePlayer.js +++ b/frontend/app/components/Session/LivePlayer.js @@ -31,7 +31,7 @@ const InitLoader = connectPlayer(state => ({ }))(Loader); -function WebPlayer ({ showAssist, session, toggleFullscreen, closeBottomBlock, live, fullscreen, jwt, loadingCredentials, assistCredendials, request }) { +const WebPlayer = React.memo(({ showAssist, session, toggleFullscreen, closeBottomBlock, live, fullscreen, jwt, loadingCredentials, assistCredendials, request }) => { useEffect(() => { if (!loadingCredentials) { initPlayer(session, jwt, assistCredendials); @@ -60,7 +60,7 @@ function WebPlayer ({ showAssist, session, toggleFullscreen, closeBottomBlock, l ); -} +}); export default withRequest({ initialData: null, diff --git a/frontend/app/components/Session/Session.js b/frontend/app/components/Session/Session.js index 6b90ff4c1..8ff629c55 100644 --- a/frontend/app/components/Session/Session.js +++ b/frontend/app/components/Session/Session.js @@ -6,6 +6,7 @@ import { fetchList as fetchSlackList } from 'Duck/integrations/slack'; import { Link, NoContent, Loader } from 'UI'; import { sessions as sessionsRoute } from 'App/routes'; import withPermissions from 'HOCs/withPermissions' +import { fetchLiveList } from 'Duck/sessions'; import LivePlayer from './LivePlayer'; import WebPlayer from './WebPlayer'; @@ -20,6 +21,8 @@ function Session({ session, fetchSession, fetchSlackList, + fetchLiveList, + filters }) { usePageTitle("OpenReplay Session Player"); useEffect(() => { @@ -36,6 +39,12 @@ function Session({ } },[ sessionId ]); + // useEffect(() => { + // if (session && session.live) { + // fetchLiveList(filters.toJS()) + // } + // }, [session]) + return ( ({ list: state.fetchList, })) +@connect(state => ({ + timelinePointer: state.getIn(['sessions', 'timelinePointer']), +})) export default class Fetch extends React.PureComponent { state = { filter: "", + filteredList: this.props.list, current: null, + currentIndex: 0, + showFetchDetails: false, + hasNextError: false, + hasPreviousError: false, } - onFilterChange = (e, { value }) => this.setState({ filter: value }) + + onFilterChange = (e, { value }) => { + const { list } = this.props; + const filterRE = getRE(value, 'i'); + const filtered = list + .filter((r) => filterRE.test(r.name) || filterRE.test(r.url) || filterRE.test(r.method) || filterRE.test(r.status)); + this.setState({ filter: value, filteredList: value ? filtered : list, currentIndex: 0 }); + } setCurrent = (item, index) => { pause() + jump(item.time) this.setState({ current: item, currentIndex: index }); } - closeModal = () => this.setState({ current: null}) + onRowClick = (item, index) => { + pause() + this.setState({ current: item, currentIndex: index, showFetchDetails: true }); + } + + closeModal = () => this.setState({ current: null, showFetchDetails: false }); nextClickHander = () => { const { list } = this.props; @@ -33,6 +54,7 @@ export default class Fetch extends React.PureComponent { if (currentIndex === list.length - 1) return; const newIndex = currentIndex + 1; this.setCurrent(list[newIndex], newIndex); + this.setState({ showFetchDetails: true }); } prevClickHander = () => { @@ -42,15 +64,24 @@ export default class Fetch extends React.PureComponent { if (currentIndex === 0) return; const newIndex = currentIndex - 1; this.setCurrent(list[newIndex], newIndex); + this.setState({ showFetchDetails: true }); + } + + static getDerivedStateFromProps(nextProps, prevState) { + const { filteredList } = prevState; + if (nextProps.timelinePointer) { + let activeItem = filteredList.find((r) => r.time >= nextProps.timelinePointer.time); + activeItem = activeItem || filteredList[filteredList.length - 1]; + return { + current: activeItem, + currentIndex: filteredList.indexOf(activeItem), + }; + } } render() { - const { list } = this.props; - const { filter, current, currentIndex } = this.state; - const filterRE = getRE(filter, 'i'); - const filtered = list - .filter((r) => filterRE.test(r.name) || filterRE.test(r.url) || filterRE.test(r.method) || filterRE.test(r.status)); - + // const { list } = this.props; + const { current, currentIndex, showFetchDetails, filteredList } = this.state; return ( } - isDisplayed={ current != null } - content={ current && + isDisplayed={ current != null && showFetchDetails } + content={ current && showFetchDetails && } onClose={ this.closeModal } @@ -88,25 +119,31 @@ export default class Fetch extends React.PureComponent {

Fetch

- +
+ {/*
+
Prev
+
Next
+
*/} + +
{[ @@ -120,7 +157,7 @@ export default class Fetch extends React.PureComponent { width: 60, }, { label: "Name", - width: 130, + width: 180, render: renderName, }, { diff --git a/frontend/app/components/Session_/Network/Network.js b/frontend/app/components/Session_/Network/Network.js index 878266bda..1d86394c5 100644 --- a/frontend/app/components/Session_/Network/Network.js +++ b/frontend/app/components/Session_/Network/Network.js @@ -3,14 +3,9 @@ import { connectPlayer, jump, pause } from 'Player'; import { QuestionMarkHint, Popup, Tabs, Input } from 'UI'; import { getRE } from 'App/utils'; import { TYPES } from 'Types/session/resource'; -import { formatBytes } from 'App/utils'; -import { formatMs } from 'App/date'; - -import TimeTable from '../TimeTable'; -import BottomBlock from '../BottomBlock'; -import InfoLine from '../BottomBlock/InfoLine'; import stl from './network.css'; import NetworkContent from './NetworkContent'; +import { connect } from 'react-redux'; const ALL = 'ALL'; const XHR = 'xhr'; @@ -28,73 +23,24 @@ const TAB_TO_TYPE_MAP = { [ MEDIA ]: TYPES.MEDIA, [ OTHER ]: TYPES.OTHER } -const TABS = [ ALL, XHR, JS, CSS, IMG, MEDIA, OTHER ].map(tab => ({ - text: tab, - key: tab, -})); - -const DOM_LOADED_TIME_COLOR = "teal"; -const LOAD_TIME_COLOR = "red"; export function renderName(r) { return ( - { r.name } } - content={
{ r.url }
} - size="mini" - position="right center" - /> - ); -} - -const renderXHRText = () => ( - - {XHR} - - Use our Fetch plugin - {' to capture HTTP requests and responses, including status codes and bodies.'}
- We also provide support for GraphQL - {' for easy debugging of your queries.'} - - } - className="ml-1" - /> - -); - -function renderSize(r) { - let triggerText; - let content; - if (r.decodedBodySize == null) { - triggerText = "x"; - content = "Not captured"; - } else { - const headerSize = r.headerSize || 0; - const encodedSize = r.encodedBodySize || 0; - const transferred = headerSize + encodedSize; - const showTransferred = r.headerSize != null; - - triggerText = formatBytes(r.decodedBodySize); - content = ( -
    - { showTransferred && -
  • {`${formatBytes( r.encodedBodySize + headerSize )} transfered over network`}
    • - } -
  • {`Resource size: ${formatBytes(r.decodedBodySize)} `}
    • -
- ); - } - - return ( - { triggerText } } - content={ content } - size="mini" - position="right center" - /> +
+ { r.name }
} + content={
{ r.url }
} + size="mini" + position="right center" + /> +
{ + e.stopPropagation(); + jump(r.time) + }} + >Jump
+ ); } @@ -130,14 +76,18 @@ export function renderDuration(r) { resources: state.resourceList, domContentLoadedTime: state.domContentLoadedTime, loadTime: state.loadTime, - time: state.time, + // time: state.time, playing: state.playing, domBuildingTime: state.domBuildingTime, fetchPresented: state.fetchList.length > 0, })) +@connect(state => ({ + timelinePointer: state.getIn(['sessions', 'timelinePointer']), +})) export default class Network extends React.PureComponent { state = { filter: '', + filteredList: this.props.resources, activeTab: ALL, currentIndex: 0 } @@ -149,7 +99,30 @@ export default class Network extends React.PureComponent { } onTabClick = activeTab => this.setState({ activeTab }) - onFilterChange = (e, { value }) => this.setState({ filter: value }) + + onFilterChange = (e, { value }) => { + const { resources } = this.props; + const filterRE = getRE(value, 'i'); + const filtered = resources.filter(({ type, name }) => + filterRE.test(name) && (activeTab === ALL || type === TAB_TO_TYPE_MAP[ activeTab ])); + + this.setState({ filter: value, filteredList: value ? filtered : resources, currentIndex: 0 }); + } + // onFilterChange = (e, { value }) => this.setState({ filter: value }) + + componentDidUpdate() { + console.log('test') + } + + static getDerivedStateFromProps(nextProps, prevState) { + const { filteredList } = prevState; + if (nextProps.timelinePointer) { + const activeItem = filteredList.find((r) => r.time >= nextProps.timelinePointer.time); + return { + currentIndex: activeItem ? filteredList.indexOf(activeItem) : filteredList.length - 1, + }; + } + } render() { const { @@ -159,50 +132,23 @@ export default class Network extends React.PureComponent { loadTime, domBuildingTime, fetchPresented, - time, + // time, playing } = this.props; - const { filter, activeTab, currentIndex } = this.state; - const filterRE = getRE(filter, 'i'); - let filtered = resources.filter(({ type, name }) => - filterRE.test(name) && (activeTab === ALL || type === TAB_TO_TYPE_MAP[ activeTab ])); - -// const referenceLines = []; -// if (domContentLoadedTime != null) { -// referenceLines.push({ -// time: domContentLoadedTime, -// color: DOM_LOADED_TIME_COLOR, -// }) -// } -// if (loadTime != null) { -// referenceLines.push({ -// time: loadTime, -// color: LOAD_TIME_COLOR, -// }) -// } -// -// let tabs = TABS; -// if (!fetchPresented) { -// tabs = TABS.map(tab => tab.key === XHR -// ? { -// text: renderXHRText(), -// key: XHR, -// } -// : tab -// ); -// } - - const resourcesSize = filtered.reduce((sum, { decodedBodySize }) => sum + (decodedBodySize || 0), 0); - const transferredSize = filtered + const { filter, activeTab, currentIndex, filteredList } = this.state; + // const filterRE = getRE(filter, 'i'); + // let filtered = resources.filter(({ type, name }) => + // filterRE.test(name) && (activeTab === ALL || type === TAB_TO_TYPE_MAP[ activeTab ])); + const resourcesSize = filteredList.reduce((sum, { decodedBodySize }) => sum + (decodedBodySize || 0), 0); + const transferredSize = filteredList .reduce((sum, { headerSize, encodedBodySize }) => sum + (headerSize || 0) + (encodedBodySize || 0), 0); return ( - {/* - - - - - - - - 0 } - /> - 0 } - /> - - - - - - {[ - { - label: "Status", - dataKey: 'status', - width: 70, - }, { - label: "Type", - dataKey: 'type', - width: 60, - }, { - label: "Name", - width: 130, - render: renderName, - }, - { - label: "Size", - width: 60, - render: renderSize, - }, - { - label: "Time", - width: 80, - render: renderDuration, - } - ]} - - - */} ); } diff --git a/frontend/app/components/Session_/Network/network.css b/frontend/app/components/Session_/Network/network.css index eb37e49c9..e299211da 100644 --- a/frontend/app/components/Session_/Network/network.css +++ b/frontend/app/components/Session_/Network/network.css @@ -22,7 +22,7 @@ white-space: nowrap; text-overflow: ellipsis; overflow: hidden; - max-width: 100%; + max-width: 80%; width: fit-content; } .popupNameContent { diff --git a/frontend/app/components/Session_/Player/Controls/Timeline.js b/frontend/app/components/Session_/Player/Controls/Timeline.js index c25101d69..27c1de1ab 100644 --- a/frontend/app/components/Session_/Player/Controls/Timeline.js +++ b/frontend/app/components/Session_/Player/Controls/Timeline.js @@ -7,6 +7,7 @@ import TimeTracker from './TimeTracker'; import { ReduxTime } from './Time'; import stl from './timeline.css'; import { TYPES } from 'Types/session/event'; +import { setTimelinePointer } from 'Duck/sessions'; const getPointerIcon = (type) => { // exception, @@ -69,7 +70,7 @@ const getPointerIcon = (type) => { state.getIn([ 'sessions', 'current', 'clickRageTime' ]), returningLocationTime: state.getIn([ 'sessions', 'current', 'returningLocation' ]) && state.getIn([ 'sessions', 'current', 'returningLocationTime' ]), -})) +}), { setTimelinePointer }) export default class Timeline extends React.PureComponent { seekProgress = (e) => { const { endTime } = this.props; @@ -78,9 +79,10 @@ export default class Timeline extends React.PureComponent { this.props.jump(time); } - createEventClickHandler = time => (e) => { + createEventClickHandler = pointer => (e) => { e.stopPropagation(); - this.props.jump(time) + this.props.jump(pointer.time); + this.props.setTimelinePointer(pointer); } componentDidMount() { @@ -144,7 +146,7 @@ export default class Timeline extends React.PureComponent { //width: `${ 2000 * scale }%` } } className={ stl.clickRage } - onClick={ this.createEventClickHandler(iss.time) } + onClick={ this.createEventClickHandler(iss) } >
+ { live && } { live && } { !live && ( <> diff --git a/frontend/app/components/Session_/TimeTable/TimeTable.js b/frontend/app/components/Session_/TimeTable/TimeTable.js index 9753e3d44..e316dae56 100644 --- a/frontend/app/components/Session_/TimeTable/TimeTable.js +++ b/frontend/app/components/Session_/TimeTable/TimeTable.js @@ -135,7 +135,7 @@ export default class TimeTable extends React.PureComponent { ...computeTimeLine(this.props.rows, this.state.firstVisibleRowIndex, this.visibleCount), }); } - if (this.props.activeIndex && prevProps.activeIndex !== this.props.activeIndex && this.scroller.current != null) { + if (this.props.activeIndex >= 0 && prevProps.activeIndex !== this.props.activeIndex && this.scroller.current != null) { this.scroller.current.scrollToRow(this.props.activeIndex); } } @@ -168,7 +168,7 @@ export default class TimeTable extends React.PureComponent {
onRowClick(row, index) : null } id="table-row" > @@ -223,7 +223,7 @@ export default class TimeTable extends React.PureComponent { navigation=false, referenceLines = [], additionalHeight = 0, - activeIndex + activeIndex, } = this.props; const { timewidth, @@ -247,7 +247,7 @@ export default class TimeTable extends React.PureComponent { return (
{ navigation && -
+
{ @@ -242,13 +244,16 @@ const reducer = (state = initialState, action = {}) => { return state.set('insights', List(action.data).sort((a, b) => b.count - a.count)); case SET_FUNNEL_PAGE_FLAG: return state.set('funnelPage', action.funnelPage ? Map(action.funnelPage) : false); + case SET_TIMELINE_POINTER: + return state.set('timelinePointer', action.pointer); default: return state; } }; export default withRequestState({ - _: [ FETCH, FETCH_LIST, FETCH_LIVE_LIST ], + _: [ FETCH, FETCH_LIST ], + fetchLiveListRequest: FETCH_LIVE_LIST, fetchFavoriteListRequest: FETCH_FAVORITE_LIST, toggleFavoriteRequest: TOGGLE_FAVORITE, fetchErrorStackList: FETCH_ERROR_STACK, @@ -262,10 +267,10 @@ function init(session) { } } -export const fetchList = (params = {}, clear = false) => (dispatch, getState) => { +export const fetchList = (params = {}, clear = false, live = false) => (dispatch, getState) => { const activeTab = getState().getIn([ 'sessions', 'activeTab' ]); - return dispatch(activeTab && activeTab.type === 'live' ? { + return dispatch((activeTab && activeTab.type === 'live' || live )? { types: FETCH_LIVE_LIST.toArray(), call: client => client.post('/assist/sessions', params), } : { @@ -376,3 +381,9 @@ export function setFunnelPage(funnelPage) { } } +export function setTimelinePointer(pointer) { + return { + type: SET_TIMELINE_POINTER, + pointer + } +} \ No newline at end of file diff --git a/frontend/app/svg/icons/os/fedora.svg b/frontend/app/svg/icons/os/fedora.svg new file mode 100644 index 000000000..86e9f115e --- /dev/null +++ b/frontend/app/svg/icons/os/fedora.svg @@ -0,0 +1,5 @@ + + + Fedora logo (2021) + + diff --git a/frontend/app/types/filter/filter.js b/frontend/app/types/filter/filter.js index 42637a2fa..8b3301aad 100644 --- a/frontend/app/types/filter/filter.js +++ b/frontend/app/types/filter/filter.js @@ -34,8 +34,8 @@ export default Record({ startDate, endDate, - sort: undefined, - order: undefined, + sort: 'startTs', + order: 'desc', viewed: undefined, consoleLogCount: undefined, From 1c1a1a7a27c997ef5921baf241f050b07d57c372 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 24 Dec 2021 16:19:15 +0530 Subject: [PATCH 011/206] feat(ui) - assist tabs show laoder --- .../components/AssistTabs/AssistTabs.tsx | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx index 8f0641200..1b1d694dd 100644 --- a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx +++ b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx @@ -3,7 +3,7 @@ import { connect } from 'react-redux'; import { applyFilter, addAttribute } from 'Duck/filters'; import { fetchList } from 'Duck/sessions'; import { KEYS } from 'Types/filter/customFilter'; -import { Link } from 'UI'; +import { Link, Loader } from 'UI'; import Filter from 'Types/filter'; import { List } from 'immutable'; import Counter from 'App/components/shared/SessionItem/Counter'; @@ -34,6 +34,13 @@ interface Props { const AssistTabs = React.memo((props: Props) => { const [showMenu, setShowMenu] = useState(false) + + useEffect(() => { + if (showMenu) { + props.fetchLiveList(); + } + }, [showMenu]) + useEffect(() => { if (!props.loading && props.list.size === 0) { props.fetchLiveList(); @@ -48,9 +55,11 @@ const AssistTabs = React.memo((props: Props) => { className="border z-10 absolute bg-white rounded shadow right-0" style={{ minWidth: "180px"}} > - {props.list.map((item, index) => ( - - ))} + + {props.list.map((item, index) => ( + + ))} +
)}
@@ -60,7 +69,7 @@ const AssistTabs = React.memo((props: Props) => { export default connect(state => { const session = state.getIn([ 'sessions', 'current' ]); return { - loading: state.getIn([ 'sessions', 'loading' ]), + loading: state.getIn([ 'sessions', 'fetchLiveListRequest', 'loading' ]), list: state.getIn(['sessions', 'liveSessions']).filter(i => i.userId === session.userId), session, filters: state.getIn([ 'filters', 'appliedFilter' ]), From 3eec195d794c987440d624eac9235d13327bc2bc Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 24 Dec 2021 16:38:54 +0530 Subject: [PATCH 012/206] fix(ui) - check for ee --- frontend/app/components/Session/LivePlayer.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Session/LivePlayer.js b/frontend/app/components/Session/LivePlayer.js index cbc3410cd..c67103185 100644 --- a/frontend/app/components/Session/LivePlayer.js +++ b/frontend/app/components/Session/LivePlayer.js @@ -31,7 +31,7 @@ const InitLoader = connectPlayer(state => ({ }))(Loader); -function WebPlayer ({ showAssist, session, toggleFullscreen, closeBottomBlock, live, fullscreen, jwt, loadingCredentials, assistCredendials, request }) { +function WebPlayer ({ showAssist, session, toggleFullscreen, closeBottomBlock, live, fullscreen, jwt, loadingCredentials, assistCredendials, request, isEnterprise }) { useEffect(() => { if (!loadingCredentials) { initPlayer(session, jwt, assistCredendials); @@ -41,7 +41,9 @@ function WebPlayer ({ showAssist, session, toggleFullscreen, closeBottomBlock, l // LAYOUT (TODO: local layout state - useContext or something..) useEffect(() => { - request(); + if (isEnterprise) { + request(); + } return () => { toggleFullscreen(false); closeBottomBlock(); @@ -74,6 +76,7 @@ export default withRequest({ showAssist: state.getIn([ 'sessions', 'showChatWindow' ]), jwt: state.get('jwt'), fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), + isEnterprise: state.getIn([ 'user', 'client', 'edition' ]) === 'ee', }), { toggleFullscreen, closeBottomBlock }, )(WebPlayer))); \ No newline at end of file From 54677af4fd6791d54a3af59622d40cb15b7224cb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 24 Dec 2021 12:30:28 +0100 Subject: [PATCH 013/206] feat(api): sessions-search support null results feat(api): pg_client support timeout feat(api): pg_client support pool config feat(api): pg_client support long-queries --- api/.env.default | 2 ++ api/chalicelib/core/sessions.py | 8 ++++---- api/chalicelib/core/weekly_report.py | 5 +++-- api/chalicelib/utils/pg_client.py | 21 +++++++++++++++------ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/api/.env.default b/api/.env.default index 2e9101b60..b705fb339 100644 --- a/api/.env.default +++ b/api/.env.default @@ -35,6 +35,8 @@ pg_host=postgresql.db.svc.cluster.local pg_password=asayerPostgres pg_port=5432 pg_user=postgres +pg_timeout=30 +pg_minconn=50 put_S3_TTL=20 sentryURL= sessions_bucket=mobs diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 7a00de9bf..d5f574b8b 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -562,7 +562,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f {query_part};""", full_args) else: - main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200) AS sessions + main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200), '[]'::JSONB) AS sessions FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {query_part} @@ -825,7 +825,7 @@ def get_session_ids_by_user_ids(project_id, user_ids): def delete_sessions_by_session_ids(session_ids): - with pg_client.PostgresClient() as cur: + with pg_client.PostgresClient(long_query=True) as cur: query = cur.mogrify( """\ DELETE FROM public.sessions @@ -839,7 +839,7 @@ def delete_sessions_by_session_ids(session_ids): def delete_sessions_by_user_ids(project_id, user_ids): - with pg_client.PostgresClient() as cur: + with pg_client.PostgresClient(long_query=True) as cur: query = cur.mogrify( """\ DELETE FROM public.sessions @@ -853,6 +853,6 @@ def delete_sessions_by_user_ids(project_id, user_ids): def count_all(): - with pg_client.PostgresClient() as cur: + with pg_client.PostgresClient(long_query=True) as cur: row = cur.execute(query="SELECT COUNT(session_id) AS count FROM public.sessions") return row.get("count", 0) diff --git a/api/chalicelib/core/weekly_report.py b/api/chalicelib/core/weekly_report.py index cb4fab769..ed8cf29d1 100644 --- a/api/chalicelib/core/weekly_report.py +++ b/api/chalicelib/core/weekly_report.py @@ -1,6 +1,7 @@ +from decouple import config + from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -from decouple import config from chalicelib.utils.helper import get_issue_title LOWEST_BAR_VALUE = 3 @@ -30,7 +31,7 @@ def edit_config(user_id, weekly_report): def cron(): - with pg_client.PostgresClient() as cur: + with pg_client.PostgresClient(long_query=True) as cur: params = {"3_days_ago": TimeUTC.midnight(delta_days=-3), "1_week_ago": TimeUTC.midnight(delta_days=-7), "2_week_ago": TimeUTC.midnight(delta_days=-14), diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 8fb869367..097ecfafe 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -1,15 +1,17 @@ +from threading import Semaphore + import psycopg2 import psycopg2.extras from decouple import config +from psycopg2 import pool PG_CONFIG = {"host": config("pg_host"), "database": config("pg_dbname"), "user": config("pg_user"), "password": config("pg_password"), "port": config("pg_port", cast=int)} - -from psycopg2 import pool -from threading import Semaphore +if config("pg_timeout", cast=int, default=0) > 0: + PG_CONFIG["options"] = f"-c statement_timeout={config('pg_timeout', cast=int) * 1000}" class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool): @@ -27,7 +29,7 @@ class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool): try: - postgreSQL_pool = ORThreadedConnectionPool(50, 100, **PG_CONFIG) + postgreSQL_pool = ORThreadedConnectionPool(config("pg_minconn", cast=int, default=20), 100, **PG_CONFIG) if (postgreSQL_pool): print("Connection pool created successfully") except (Exception, psycopg2.DatabaseError) as error: @@ -38,9 +40,14 @@ except (Exception, psycopg2.DatabaseError) as error: class PostgresClient: connection = None cursor = None + long_query = False - def __init__(self): - self.connection = postgreSQL_pool.getconn() + def __init__(self, long_query=False): + self.long_query = long_query + if long_query: + self.connection = psycopg2.connect(**PG_CONFIG) + else: + self.connection = postgreSQL_pool.getconn() def __enter__(self): if self.cursor is None: @@ -51,6 +58,8 @@ class PostgresClient: try: self.connection.commit() self.cursor.close() + if self.long_query: + self.connection.close() except Exception as error: print("Error while committing/closing PG-connection", error) raise error From 7349e7c560f583cf79bbe5d8be3e7aab8c257583 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 24 Dec 2021 15:52:46 +0100 Subject: [PATCH 014/206] feat(api): sessions-search by issue --- api/chalicelib/core/sessions.py | 7 ++++ api/schemas.py | 33 +++++++++++++++++-- .../db/init_dbs/postgresql/init_schema.sql | 3 -- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index d5f574b8b..3981ff990 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -281,6 +281,13 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints.append( _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: diff --git a/api/schemas.py b/api/schemas.py index ec8586cb7..9ad5c3295 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -1,7 +1,7 @@ from enum import Enum from typing import Optional, List, Literal, Union -from pydantic import BaseModel, Field, EmailStr, HttpUrl +from pydantic import BaseModel, Field, EmailStr, HttpUrl, root_validator from chalicelib.utils.TimeUTC import TimeUTC @@ -380,6 +380,7 @@ class FilterType(str, Enum): duration = "DURATION" platform = "PLATFORM" metadata = "METADATA" + issue = "ISSUE" class SearchEventOperator(str, Enum): @@ -406,6 +407,21 @@ class SearchEventOrder(str, Enum): _and = "and" +class IssueType(str, Enum): + click_rage = 'click_rage' + dead_click = 'dead_click' + excessive_scrolling = 'excessive_scrolling' + bad_request = 'bad_request' + missing_resource = 'missing_resource' + memory = 'memory' + cpu = 'cpu' + slow_resource = 'slow_resource' + slow_page_load = 'slow_page_load' + crash = 'crash' + custom = 'custom' + js_exception = 'js_exception' + + class _SessionSearchEventSchema(BaseModel): custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) @@ -418,11 +434,22 @@ class _SessionSearchEventSchema(BaseModel): class _SessionSearchFilterSchema(BaseModel): custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) - value: Union[Optional[Union[str, int]], Optional[List[Union[str, int]]]] = Field(...) + value: Union[Optional[Union[str, int, IssueType, PlatformType]], + Optional[List[Union[str, int, IssueType, PlatformType]]]] = Field(...) type: FilterType = Field(...) operator: SearchEventOperator = Field(...) source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) + @root_validator + def check_card_number_omitted(cls, values): + if values.get("type") == FilterType.issue: + for v in values.get("value"): + assert IssueType(v) + elif values.get("type") == FilterType.platform: + for v in values.get("value"): + assert PlatformType(v) + return values + class SessionsSearchPayloadSchema(BaseModel): events: List[_SessionSearchEventSchema] = Field([]) @@ -439,10 +466,12 @@ class SessionsSearchPayloadSchema(BaseModel): class Config: alias_generator = attribute_to_camel_case + class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) order: Optional[str] = Field(default=None) + class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): range_value: Optional[str] = Field(None) sort: Optional[str] = Field(None) diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 345bf6b2c..7d3068a42 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -523,8 +523,6 @@ $$ metadata_8 text DEFAULT NULL, metadata_9 text DEFAULT NULL, metadata_10 text DEFAULT NULL --- , --- rehydration_id integer REFERENCES rehydrations(rehydration_id) ON DELETE SET NULL ); CREATE INDEX ON sessions (project_id, start_ts); CREATE INDEX ON sessions (project_id, user_id); @@ -542,7 +540,6 @@ $$ CREATE INDEX ON sessions (project_id, metadata_8); CREATE INDEX ON sessions (project_id, metadata_9); CREATE INDEX ON sessions (project_id, metadata_10); --- CREATE INDEX ON sessions (rehydration_id); CREATE INDEX ON sessions (project_id, watchdogs_score DESC); CREATE INDEX platform_idx ON public.sessions (platform); From 7c4b410a359c162b4e428da3dd1c2e3abc93d680 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Fri, 24 Dec 2021 22:45:27 +0530 Subject: [PATCH 015/206] fix(ui) - show live sessions by user --- .../components/AssistTabs/AssistTabs.tsx | 76 ++++--------------- .../components/SessionList/SessionList.tsx | 40 ++++++++++ .../Assist/components/SessionList/index.ts | 1 + .../components/Session_/PlayerBlockHeader.js | 2 +- 4 files changed, 55 insertions(+), 64 deletions(-) create mode 100644 frontend/app/components/Assist/components/SessionList/SessionList.tsx create mode 100644 frontend/app/components/Assist/components/SessionList/index.ts diff --git a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx index 1b1d694dd..bd026797e 100644 --- a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx +++ b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx @@ -1,77 +1,27 @@ import React, { useEffect, useState } from 'react'; -import { connect } from 'react-redux'; -import { applyFilter, addAttribute } from 'Duck/filters'; -import { fetchList } from 'Duck/sessions'; -import { KEYS } from 'Types/filter/customFilter'; -import { Link, Loader } from 'UI'; -import Filter from 'Types/filter'; -import { List } from 'immutable'; -import Counter from 'App/components/shared/SessionItem/Counter'; -import { fetchLiveList } from 'Duck/sessions'; -import { session as sessionRoute } from 'App/routes'; -import { session } from 'App/components/Session_/session.css'; - -const RowItem = ({ startedAt, sessionId }) => { - return ( - -
- Tab1 - -
- - ); -} +import { SlideModal } from 'UI'; +import SessionList from '../SessionList'; interface Props { - list: List, - session: any, - fetchLiveList: () => void, - applyFilter: () => void, - filters: Filter - addAttribute: (obj) => void, - loading: boolean + userId: any, } const AssistTabs = React.memo((props: Props) => { const [showMenu, setShowMenu] = useState(false) - useEffect(() => { - if (showMenu) { - props.fetchLiveList(); - } - }, [showMenu]) - - useEffect(() => { - if (!props.loading && props.list.size === 0) { - props.fetchLiveList(); - } - }, [props.list]) - return (
-
setShowMenu(!showMenu)}>Active Tabs
- {showMenu && ( -
- - {props.list.map((item, index) => ( - - ))} - -
- )} +
setShowMenu(!showMenu)}> + Live Sessions +
+ Live Sessions by {props.userId}
} + isDisplayed={ showMenu } + content={ showMenu && } + onClose={ () => setShowMenu(false) } + />
); }); -export default connect(state => { - const session = state.getIn([ 'sessions', 'current' ]); - return { - loading: state.getIn([ 'sessions', 'fetchLiveListRequest', 'loading' ]), - list: state.getIn(['sessions', 'liveSessions']).filter(i => i.userId === session.userId), - session, - filters: state.getIn([ 'filters', 'appliedFilter' ]), - } -}, { applyFilter, addAttribute, fetchLiveList })(AssistTabs); \ No newline at end of file +export default AssistTabs; \ No newline at end of file diff --git a/frontend/app/components/Assist/components/SessionList/SessionList.tsx b/frontend/app/components/Assist/components/SessionList/SessionList.tsx new file mode 100644 index 000000000..aaab06fca --- /dev/null +++ b/frontend/app/components/Assist/components/SessionList/SessionList.tsx @@ -0,0 +1,40 @@ +import React, { useEffect } from 'react'; +import { connect } from 'react-redux'; +import { fetchLiveList } from 'Duck/sessions'; +import { Loader, NoContent } from 'UI'; +import SessionItem from 'Shared/SessionItem'; + +interface Props { + loading: boolean, + list: any, + session: any, + fetchLiveList: () => void, +} +function SessionList(props: Props) { + useEffect(() => { + props.fetchLiveList(); + }, []) + + return ( + + +
+ { props.list.map(session => ) } +
+ + + ); +} + +export default connect(state => { + const session = state.getIn([ 'sessions', 'current' ]); + return { + session, + list: state.getIn(['sessions', 'liveSessions']) + .filter(i => i.userId === session.userId && i.sessionId !== session.sessionId), + loading: state.getIn([ 'sessions', 'fetchLiveListRequest', 'loading' ]), + } +}, { fetchLiveList })(SessionList); \ No newline at end of file diff --git a/frontend/app/components/Assist/components/SessionList/index.ts b/frontend/app/components/Assist/components/SessionList/index.ts new file mode 100644 index 000000000..779c9df2a --- /dev/null +++ b/frontend/app/components/Assist/components/SessionList/index.ts @@ -0,0 +1 @@ +export { default } from './SessionList'; \ No newline at end of file diff --git a/frontend/app/components/Session_/PlayerBlockHeader.js b/frontend/app/components/Session_/PlayerBlockHeader.js index fbc0b0a5b..20b5af366 100644 --- a/frontend/app/components/Session_/PlayerBlockHeader.js +++ b/frontend/app/components/Session_/PlayerBlockHeader.js @@ -116,7 +116,7 @@ export default class PlayerBlockHeader extends React.PureComponent {
- { live && } + { live && } { live && } { !live && ( <> From adee10929f3138070b95e6c67ab41dd177979795 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 24 Dec 2021 21:01:30 +0100 Subject: [PATCH 016/206] feat(api): sessions-search TIME_BETWEEN_EVENTS --- api/chalicelib/core/sessions.py | 68 +++++++++++++++++++++++++++------ api/schemas.py | 46 +++++++++++++++++++--- 2 files changed, 97 insertions(+), 17 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 3981ff990..70858481c 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1,6 +1,6 @@ import schemas from chalicelib.core import events, sessions_metas, metadata, events_ios, \ - sessions_mobs, issues, projects, errors, resources, assist + sessions_mobs, issues, projects, errors, resources, assist, performance_event from chalicelib.utils import pg_client, helper, dev SESSION_PROJECTION_COLS = """s.project_id, @@ -318,16 +318,16 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f "main.session_id=event_0.session_id"] if data.events_order == schemas.SearchEventOrder._then: event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") - - event.value = helper.values_for_operator(value=event.value, op=event.operator) - # event_args = _multiple_values(event.value) e_k = f"e_value{i}" - full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} - if event_type not in list(events.SUPPORTED_TYPES.keys()) \ - or event.value in [None, "", "*"] \ - and (event_type != events.event_type.ERROR.ui_type \ - or event_type != events.event_type.ERROR_IOS.ui_type): - continue + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} + + # if event_type not in list(events.SUPPORTED_TYPES.keys()) \ + # or event.value in [None, "", "*"] \ + # and (event_type != events.event_type.ERROR.ui_type \ + # or event_type != events.event_type.ERROR_IOS.ui_type): + # continue if event_type == events.event_type.CLICK.ui_type: event_from = event_from % f"{events.event_type.CLICK.table} AS main " if not is_any: @@ -432,6 +432,52 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where.append( _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", event.value, value_key=e_k)) + elif event_type == [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + event_where.append( + _multiple_conditions( + f"main.{performance_event.get_col(event_type)} {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, op=event.operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, op=event.operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) + else: continue @@ -464,7 +510,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f """) else: events_query_from.append(f"""\ - (SELECT main.session_id, MIN(timestamp) AS timestamp + (SELECT main.session_id, MIN(main.timestamp) AS timestamp FROM {event_from} WHERE {" AND ".join(event_where)} GROUP BY 1 diff --git a/api/schemas.py b/api/schemas.py index 9ad5c3295..47966c854 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional, List, Literal, Union +from typing import Optional, List, Union from pydantic import BaseModel, Field, EmailStr, HttpUrl, root_validator @@ -312,10 +312,18 @@ class AlertColumn(str, Enum): errors__backend__count = "errors.backend.count" +class MathOperator(str, Enum): + _less = "<" + _greater = ">" + _less_eq = "<=" + _greater_eq = ">=" + + class _AlertQuerySchema(BaseModel): left: AlertColumn = Field(...) right: float = Field(...) - operator: Literal["<", ">", "<=", ">="] = Field(...) + # operator: Literal["<", ">", "<=", ">="] = Field(...) + operator: MathOperator = Field(...) class AlertSchema(BaseModel): @@ -360,6 +368,12 @@ class EventType(str, Enum): error_ios = "ERROR_IOS" +class PerformanceEventType(str, Enum): + location_dom_complete = "DOM_COMPLETE" + location_largest_contentful_paint_time = "LARGEST_CONTENTFUL_PAINT_TIME" + time_between_events = "TIME_BETWEEN_EVENTS" + + class FilterType(str, Enum): user_os = "USEROS" user_browser = "USERBROWSER" @@ -399,6 +413,7 @@ class SearchEventOperator(str, Enum): class PlatformType(str, Enum): mobile = "mobile" desktop = "desktop" + tablet = "tablet" class SearchEventOrder(str, Enum): @@ -422,14 +437,33 @@ class IssueType(str, Enum): js_exception = 'js_exception' -class _SessionSearchEventSchema(BaseModel): - custom: Optional[List[str]] = Field(None) +class _SessionSearchEventRaw(BaseModel): + custom: Optional[List[Union[str, int]]] = Field(None) + customOperator: Optional[MathOperator] = Field(None) key: Optional[str] = Field(None) - value: Union[Optional[str], Optional[List[str]]] = Field(...) - type: EventType = Field(...) + value: Union[str, List[str]] = Field(...) + type: Union[EventType, PerformanceEventType] = Field(...) operator: SearchEventOperator = Field(...) source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) + @root_validator + def check_card_number_omitted(cls, values): + if isinstance(values.get("type"), PerformanceEventType): + assert values.get("custom") is not None, "custom should not be None for PerformanceEventType" + assert values.get("customOperator") is not None \ + , "customOperator should not be None for PerformanceEventType" + if values["type"] == PerformanceEventType.time_between_events: + assert len(values.get("value", [])) == 2, \ + f"must provide 2 Events as value for {PerformanceEventType.time_between_events}" + assert isinstance(values["value"][0], _SessionSearchEventRaw) \ + and isinstance(values["value"][1], _SessionSearchEventRaw) \ + , f"event should be of type _SessionSearchEventRaw for {PerformanceEventType.time_between_events}" + return values + + +class _SessionSearchEventSchema(_SessionSearchEventRaw): + value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...) + class _SessionSearchFilterSchema(BaseModel): custom: Optional[List[str]] = Field(None) From 22134e49a872d1d35402dee097d54a34513fbc43 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 24 Dec 2021 21:06:40 +0100 Subject: [PATCH 017/206] feat(api): perf module --- api/chalicelib/core/performance_event.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 api/chalicelib/core/performance_event.py diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py new file mode 100644 index 000000000..d8cb2da6b --- /dev/null +++ b/api/chalicelib/core/performance_event.py @@ -0,0 +1,8 @@ +import schemas + + +def get_col(perf: schemas.PerformanceEventType): + return { + schemas.PerformanceEventType.location_dom_complete: "dom_building_time", + # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" + }.get(perf) From 06769cbda0697e7a4692a9b8b690b1d8de0773d9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 12:49:18 +0100 Subject: [PATCH 018/206] feat(DB): events.pages delta to match FOSS&EE --- .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 1 - .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 22 +++++++++++++++++++ .../db/init_dbs/postgresql/init_schema.sql | 16 +++++++++++++- .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 5 ----- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 6 +++++ .../db/init_dbs/postgresql/init_schema.sql | 1 + 6 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql create mode 100644 scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql index eac07d08e..8b9fe6cfd 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -16,5 +16,4 @@ CREATE TABLE traces CREATE INDEX traces_user_id_idx ON traces (user_id); CREATE INDEX traces_tenant_id_idx ON traces (tenant_id); -CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions(user_id,session_id); COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql new file mode 100644 index 000000000..afbe2a218 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -0,0 +1,22 @@ +BEGIN; + +CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); + +-- FOS: +CREATE INDEX IF NOT EXISTS pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; +CREATE INDEX IF NOT EXISTS pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; +CREATE INDEX IF NOT EXISTS pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; +CREATE INDEX IF NOT EXISTS pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; +CREATE INDEX IF NOT EXISTS pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; +CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; +CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; +CREATE INDEX IF NOT EXISTS pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR + first_paint_time > 0 OR + dom_content_loaded_time > 0 OR + ttfb > 0 OR + time_to_interactive > 0; +CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; +CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; + + +END; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 3cf1d6751..517029893 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -724,6 +724,7 @@ $$ CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); CREATE INDEX ON events.pages (timestamp); + CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); CREATE INDEX pages_base_path_idx ON events.pages (base_path); CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); @@ -743,8 +744,21 @@ $$ CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; CREATE INDEX pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0; CREATE INDEX pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0; + CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; + CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; + CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; + CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; + CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; + CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; + CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; + CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR + first_paint_time > 0 OR + dom_content_loaded_time > 0 OR + ttfb > 0 OR + time_to_interactive > 0; + CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; + CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); - CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; CREATE TABLE events.clicks diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql index d6a3abf83..e69de29bb 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql @@ -1,5 +0,0 @@ -BEGIN; - -CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); - -END; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql new file mode 100644 index 000000000..531f99590 --- /dev/null +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -0,0 +1,6 @@ +BEGIN; + +CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); + +CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); +END; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 7d3068a42..79e846b8d 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -696,6 +696,7 @@ $$ CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); CREATE INDEX ON events.pages (timestamp); + CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); CREATE INDEX pages_base_path_idx ON events.pages (base_path); CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); From 9cc5542fc77612882e48a1160c05143dd49e7e5f Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 27 Dec 2021 20:07:08 +0530 Subject: [PATCH 019/206] change(ui) - offline vs live session --- .../components/AssistTabs/AssistTabs.tsx | 21 +++++++++--- .../components/AssistTabs/assistTabs.css | 5 +++ frontend/app/components/Session/LivePlayer.js | 7 ++-- frontend/app/components/Session/Session.js | 17 +++------- .../components/Session_/PlayerBlockHeader.js | 33 ++++++++++++------- .../components/Session_/playerBlockHeader.css | 12 +++++++ .../shared/SessionItem/SessionItem.js | 24 ++++++++++---- .../shared/SessionItem/sessionItem.css | 2 +- frontend/app/duck/funnels.js | 1 - frontend/app/duck/sessions.js | 12 ++++++- .../MessageDistributor/MessageDistributor.ts | 4 +-- frontend/app/player/singletone.js | 6 ++-- frontend/app/types/session/session.js | 3 +- 13 files changed, 99 insertions(+), 48 deletions(-) diff --git a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx index bd026797e..ff62a899a 100644 --- a/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx +++ b/frontend/app/components/Assist/components/AssistTabs/AssistTabs.tsx @@ -1,18 +1,29 @@ import React, { useEffect, useState } from 'react'; -import { SlideModal } from 'UI'; +import { SlideModal, Icon } from 'UI'; import SessionList from '../SessionList'; +import stl from './assistTabs.css' interface Props { userId: any, } -const AssistTabs = React.memo((props: Props) => { +const AssistTabs = (props: Props) => { const [showMenu, setShowMenu] = useState(false) return (
-
setShowMenu(!showMenu)}> - Live Sessions +
+
setShowMenu(!showMenu)} + > + More Live Sessions +
+ by +
+ +
{props.userId}
+
Live Sessions by {props.userId}
} @@ -22,6 +33,6 @@ const AssistTabs = React.memo((props: Props) => { />
); -}); +}; export default AssistTabs; \ No newline at end of file diff --git a/frontend/app/components/Assist/components/AssistTabs/assistTabs.css b/frontend/app/components/Assist/components/AssistTabs/assistTabs.css index e69de29bb..462879395 100644 --- a/frontend/app/components/Assist/components/AssistTabs/assistTabs.css +++ b/frontend/app/components/Assist/components/AssistTabs/assistTabs.css @@ -0,0 +1,5 @@ +.btnLink { + cursor: pointer; + color: $green; + text-decoration: underline; +} \ No newline at end of file diff --git a/frontend/app/components/Session/LivePlayer.js b/frontend/app/components/Session/LivePlayer.js index 07b8d3adc..e63578e5e 100644 --- a/frontend/app/components/Session/LivePlayer.js +++ b/frontend/app/components/Session/LivePlayer.js @@ -31,10 +31,10 @@ const InitLoader = connectPlayer(state => ({ }))(Loader); -const WebPlayer = React.memo(({ showAssist, session, toggleFullscreen, closeBottomBlock, live, fullscreen, jwt, loadingCredentials, assistCredendials, request }) => { +function WebPlayer({ showAssist, session, toggleFullscreen, closeBottomBlock, live, fullscreen, jwt, loadingCredentials, assistCredendials, request, hasSessionsPath }) { useEffect(() => { if (!loadingCredentials) { - initPlayer(session, jwt, assistCredendials); + initPlayer(session, jwt, assistCredendials, !hasSessionsPath && session.live); } return () => cleanPlayer() }, [ session.sessionId, loadingCredentials, assistCredendials ]); @@ -60,7 +60,7 @@ const WebPlayer = React.memo(({ showAssist, session, toggleFullscreen, closeBott ); -}); +}; export default withRequest({ initialData: null, @@ -74,6 +74,7 @@ export default withRequest({ showAssist: state.getIn([ 'sessions', 'showChatWindow' ]), jwt: state.get('jwt'), fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), + hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), }), { toggleFullscreen, closeBottomBlock }, )(WebPlayer))); \ No newline at end of file diff --git a/frontend/app/components/Session/Session.js b/frontend/app/components/Session/Session.js index 8ff629c55..e7af00642 100644 --- a/frontend/app/components/Session/Session.js +++ b/frontend/app/components/Session/Session.js @@ -6,7 +6,6 @@ import { fetchList as fetchSlackList } from 'Duck/integrations/slack'; import { Link, NoContent, Loader } from 'UI'; import { sessions as sessionsRoute } from 'App/routes'; import withPermissions from 'HOCs/withPermissions' -import { fetchLiveList } from 'Duck/sessions'; import LivePlayer from './LivePlayer'; import WebPlayer from './WebPlayer'; @@ -21,8 +20,7 @@ function Session({ session, fetchSession, fetchSlackList, - fetchLiveList, - filters + hasSessionsPath }) { usePageTitle("OpenReplay Session Player"); useEffect(() => { @@ -37,13 +35,7 @@ function Session({ return () => { if (!session.exists()) return; } - },[ sessionId ]); - - // useEffect(() => { - // if (session && session.live) { - // fetchLiveList(filters.toJS()) - // } - // }, [session]) + },[ sessionId, hasSessionsPath ]); return ( { session.isIOS ? - : (session.live ? : ) + : (session.live && !hasSessionsPath ? : ) } @@ -73,10 +65,9 @@ export default withPermissions(['SESSION_REPLAY'], '', true)(connect((state, pro loading: state.getIn([ 'sessions', 'loading' ]), hasErrors: !!state.getIn([ 'sessions', 'errors' ]), session: state.getIn([ 'sessions', 'current' ]), - filters: state.getIn([ 'filters', 'appliedFilter' ]), + hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), }; }, { fetchSession, fetchSlackList, - fetchLiveList, })(Session)); \ No newline at end of file diff --git a/frontend/app/components/Session_/PlayerBlockHeader.js b/frontend/app/components/Session_/PlayerBlockHeader.js index 20b5af366..ea2b63a06 100644 --- a/frontend/app/components/Session_/PlayerBlockHeader.js +++ b/frontend/app/components/Session_/PlayerBlockHeader.js @@ -4,14 +4,14 @@ import { browserIcon, osIcon, deviceTypeIcon } from 'App/iconNames'; import { formatTimeOrDate } from 'App/date'; import { sessions as sessionsRoute, funnel as funnelRoute, funnelIssue as funnelIssueRoute, withSiteId } from 'App/routes'; import { Icon, CountryFlag, IconButton, BackLink } from 'UI'; -import { toggleFavorite } from 'Duck/sessions'; +import { toggleFavorite, setSessionPath } from 'Duck/sessions'; import cn from 'classnames'; import { connectPlayer } from 'Player'; import HeaderInfo from './HeaderInfo'; import SharePopup from '../shared/SharePopup/SharePopup'; import { fetchList as fetchListIntegration } from 'Duck/integrations/actions'; -import cls from './playerBlockHeader.css'; +import stl from './playerBlockHeader.css'; import Issues from './Issues/Issues'; import Autoplay from './Autoplay'; import AssistActions from '../Assist/components/AssistActions'; @@ -38,8 +38,9 @@ function capitalise(str) { funnelRef: state.getIn(['funnels', 'navRef']), siteId: state.getIn([ 'user', 'siteId' ]), funnelPage: state.getIn(['sessions', 'funnelPage']), + hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), }), { - toggleFavorite, fetchListIntegration + toggleFavorite, fetchListIntegration, setSessionPath }) @withRouter export default class PlayerBlockHeader extends React.PureComponent { @@ -87,21 +88,24 @@ export default class PlayerBlockHeader extends React.PureComponent { userDevice, userBrowserVersion, userDeviceType, + live, }, loading, - live, + // live, disabled, jiraConfig, fullscreen, + hasSessionsPath } = this.props; - const { history, siteId } = this.props; + // const { history, siteId } = this.props; + const _live = live && !hasSessionsPath; return ( -
+
-
+
@@ -116,12 +120,17 @@ export default class PlayerBlockHeader extends React.PureComponent {
- { live && } - { live && } - { !live && ( + { live && hasSessionsPath && ( +
this.props.setSessionPath('')}> + This Session is Now Continuing Live +
+ )} + { _live && } + { _live && } + { !_live && ( <> -
+
)} - { !live && jiraConfig && jiraConfig.token && } + { !_live && jiraConfig && jiraConfig.token && }
diff --git a/frontend/app/components/Session_/playerBlockHeader.css b/frontend/app/components/Session_/playerBlockHeader.css index 325e34256..d9934ef1e 100644 --- a/frontend/app/components/Session_/playerBlockHeader.css +++ b/frontend/app/components/Session_/playerBlockHeader.css @@ -12,3 +12,15 @@ background-color: $gray-light; } +.liveSwitchButton { + cursor: pointer; + padding: 3px 8px; + border: solid thin $green; + color: $green; + border-radius: 3px; + margin-right: 10px; + &:hover { + background-color: $green; + color: white; + } +} \ No newline at end of file diff --git a/frontend/app/components/shared/SessionItem/SessionItem.js b/frontend/app/components/shared/SessionItem/SessionItem.js index 301bf4463..c1c20fe13 100644 --- a/frontend/app/components/shared/SessionItem/SessionItem.js +++ b/frontend/app/components/shared/SessionItem/SessionItem.js @@ -10,21 +10,33 @@ import { TextEllipsis } from 'UI'; import { deviceTypeIcon } from 'App/iconNames'; -import { toggleFavorite } from 'Duck/sessions'; -import { session as sessionRoute } from 'App/routes'; +import { toggleFavorite, setSessionPath } from 'Duck/sessions'; +import { session as sessionRoute, withSiteId } from 'App/routes'; import { durationFormatted, formatTimeOrDate } from 'App/date'; import stl from './sessionItem.css'; import LiveTag from 'Shared/LiveTag'; import Bookmark from 'Shared/Bookmark'; import Counter from './Counter' +import { withRouter } from 'react-router-dom'; const Label = ({ label = '', color = 'color-gray-medium'}) => (
{label}
) @connect(state => ({ - timezone: state.getIn(['sessions', 'timezone']) -}), { toggleFavorite }) + timezone: state.getIn(['sessions', 'timezone']), + isAssist: state.getIn(['sessions', 'activeTab']).type === 'live', + siteId: state.getIn([ 'user', 'siteId' ]), +}), { toggleFavorite, setSessionPath }) +@withRouter export default class SessionItem extends React.PureComponent { + + replaySession = () => { + const { history, session: { sessionId }, siteId, isAssist } = this.props; + if (!isAssist) { + this.props.setSessionPath(history.location.pathname) + } + history.push(withSiteId(sessionRoute(sessionId), siteId)) + } // eslint-disable-next-line complexity render() { const { @@ -110,9 +122,9 @@ export default class SessionItem extends React.PureComponent {
- +
- +
diff --git a/frontend/app/components/shared/SessionItem/sessionItem.css b/frontend/app/components/shared/SessionItem/sessionItem.css index 8f9824bec..cbf7bb2d1 100644 --- a/frontend/app/components/shared/SessionItem/sessionItem.css +++ b/frontend/app/components/shared/SessionItem/sessionItem.css @@ -92,7 +92,7 @@ display: flex; align-items: center; transition: all 0.2s; - /* opacity: 0; */ + cursor: pointer; &[data-viewed=true] { opacity: 1; } diff --git a/frontend/app/duck/funnels.js b/frontend/app/duck/funnels.js index 4e591237c..7dad8550a 100644 --- a/frontend/app/duck/funnels.js +++ b/frontend/app/duck/funnels.js @@ -119,7 +119,6 @@ const reducer = (state = initialState, action = {}) => { let stages = []; if (action.isRefresh) { const activeStages = state.get('activeStages'); - console.log('test', activeStages); const oldInsights = state.get('insights'); const lastStage = action.data.stages[action.data.stages.length - 1] const lastStageIndex = activeStages.toJS()[1]; diff --git a/frontend/app/duck/sessions.js b/frontend/app/duck/sessions.js index 53ec41f3e..977af85bb 100644 --- a/frontend/app/duck/sessions.js +++ b/frontend/app/duck/sessions.js @@ -8,7 +8,6 @@ import { getRE } from 'App/utils'; import { LAST_7_DAYS } from 'Types/app/period'; import { getDateRangeFromValue } from 'App/dateRange'; - const INIT = 'sessions/INIT'; const FETCH_LIST = new RequestTypes('sessions/FETCH_LIST'); @@ -26,6 +25,7 @@ const SET_AUTOPLAY_VALUES = 'sessions/SET_AUTOPLAY_VALUES'; const TOGGLE_CHAT_WINDOW = 'sessions/TOGGLE_CHAT_WINDOW'; const SET_FUNNEL_PAGE_FLAG = 'sessions/SET_FUNNEL_PAGE_FLAG'; const SET_TIMELINE_POINTER = 'sessions/SET_TIMELINE_POINTER'; +const SET_SESSION_PATH = 'sessions/SET_SESSION_PATH'; const SET_ACTIVE_TAB = 'sessions/SET_ACTIVE_TAB'; @@ -59,6 +59,7 @@ const initialState = Map({ host: '', funnelPage: Map(), timelinePointer: null, + sessionPath: '', }); const reducer = (state = initialState, action = {}) => { @@ -246,6 +247,8 @@ const reducer = (state = initialState, action = {}) => { return state.set('funnelPage', action.funnelPage ? Map(action.funnelPage) : false); case SET_TIMELINE_POINTER: return state.set('timelinePointer', action.pointer); + case SET_SESSION_PATH: + return state.set('sessionPath', action.path); default: return state; } @@ -386,4 +389,11 @@ export function setTimelinePointer(pointer) { type: SET_TIMELINE_POINTER, pointer } +} + +export function setSessionPath(path) { + return { + type: SET_SESSION_PATH, + path + } } \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.ts b/frontend/app/player/MessageDistributor/MessageDistributor.ts index c742c10b5..35ada8ce3 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.ts +++ b/frontend/app/player/MessageDistributor/MessageDistributor.ts @@ -118,7 +118,7 @@ export default class MessageDistributor extends StatedScreen { private navigationStartOffset: number = 0; private lastMessageTime: number = 0; - constructor(private readonly session: any /*Session*/, jwt: string, config) { + constructor(private readonly session: any /*Session*/, jwt: string, config, live: boolean) { super(); this.pagesManager = new PagesManager(this, this.session.isMobile) this.mouseManager = new MouseManager(this); @@ -126,7 +126,7 @@ export default class MessageDistributor extends StatedScreen { this.sessionStart = this.session.startedAt; - if (this.session.live) { + if (live) { // const sockUrl = `wss://live.openreplay.com/1/${ this.session.siteId }/${ this.session.sessionId }/${ jwt }`; // this.subscribeOnMessages(sockUrl); initListsDepr({}) diff --git a/frontend/app/player/singletone.js b/frontend/app/player/singletone.js index 619f9b02b..9d811023e 100644 --- a/frontend/app/player/singletone.js +++ b/frontend/app/player/singletone.js @@ -28,11 +28,11 @@ document.addEventListener("visibilitychange", function() { } }); -export function init(session, jwt, config) { - const live = session.live; +export function init(session, jwt, config, live = false) { + // const live = session.live; const endTime = !live && session.duration.valueOf(); - instance = new Player(session, jwt, config); + instance = new Player(session, jwt, config, live); update({ initialized: true, live, diff --git a/frontend/app/types/session/session.js b/frontend/app/types/session/session.js index 1fabc79a6..61ca9e489 100644 --- a/frontend/app/types/session/session.js +++ b/frontend/app/types/session/session.js @@ -88,6 +88,7 @@ export default Record({ ...session }) => { const duration = Duration.fromMillis(session.duration < 1000 ? 1000 : session.duration); + const durationSeconds = duration.valueOf(); const startedAt = +startTs; const userDevice = session.userDevice || session.userDeviceType || 'Other'; @@ -96,7 +97,7 @@ export default Record({ const events = List(session.events) .map(e => SessionEvent({ ...e, time: e.timestamp - startedAt })) - .filter(({ type }) => type !== TYPES.CONSOLE); + .filter(({ type, time }) => type !== TYPES.CONSOLE && time <= durationSeconds); let resources = List(session.resources) .map(Resource); From e620d28203702b739e77cae0ed0fc9ef8aa368f5 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 27 Dec 2021 21:19:10 +0530 Subject: [PATCH 020/206] change(ui) - pointer highlight --- .../app/components/Session_/Fetch/Fetch.js | 20 ++++++++++--------- .../components/Session_/Network/Network.js | 4 +++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/frontend/app/components/Session_/Fetch/Fetch.js b/frontend/app/components/Session_/Fetch/Fetch.js index 4cc3e38b7..8bc4f2117 100644 --- a/frontend/app/components/Session_/Fetch/Fetch.js +++ b/frontend/app/components/Session_/Fetch/Fetch.js @@ -2,19 +2,20 @@ import { getRE } from 'App/utils'; import { Label, NoContent, Input, SlideModal, CloseButton } from 'UI'; import { connectPlayer, pause, jump } from 'Player'; -import Autoscroll from '../Autoscroll'; +// import Autoscroll from '../Autoscroll'; import BottomBlock from '../BottomBlock'; import TimeTable from '../TimeTable'; import FetchDetails from './FetchDetails'; import { renderName, renderDuration } from '../Network'; import { connect } from 'react-redux'; +import { setTimelinePointer } from 'Duck/sessions'; @connectPlayer(state => ({ list: state.fetchList, })) @connect(state => ({ timelinePointer: state.getIn(['sessions', 'timelinePointer']), -})) +}), { setTimelinePointer }) export default class Fetch extends React.PureComponent { state = { filter: "", @@ -43,27 +44,28 @@ export default class Fetch extends React.PureComponent { onRowClick = (item, index) => { pause() this.setState({ current: item, currentIndex: index, showFetchDetails: true }); + this.props.setTimelinePointer(null); } closeModal = () => this.setState({ current: null, showFetchDetails: false }); nextClickHander = () => { - const { list } = this.props; - const { currentIndex } = this.state; + // const { list } = this.props; + const { currentIndex, filteredList } = this.state; - if (currentIndex === list.length - 1) return; + if (currentIndex === filteredList.length - 1) return; const newIndex = currentIndex + 1; - this.setCurrent(list[newIndex], newIndex); + this.setCurrent(filteredList[newIndex], newIndex); this.setState({ showFetchDetails: true }); } prevClickHander = () => { - const { list } = this.props; - const { currentIndex } = this.state; + // const { list } = this.props; + const { currentIndex, filteredList } = this.state; if (currentIndex === 0) return; const newIndex = currentIndex - 1; - this.setCurrent(list[newIndex], newIndex); + this.setCurrent(filteredList[newIndex], newIndex); this.setState({ showFetchDetails: true }); } diff --git a/frontend/app/components/Session_/Network/Network.js b/frontend/app/components/Session_/Network/Network.js index 1d86394c5..11c7b410c 100644 --- a/frontend/app/components/Session_/Network/Network.js +++ b/frontend/app/components/Session_/Network/Network.js @@ -6,6 +6,7 @@ import { TYPES } from 'Types/session/resource'; import stl from './network.css'; import NetworkContent from './NetworkContent'; import { connect } from 'react-redux'; +import { setTimelinePointer } from 'Duck/sessions'; const ALL = 'ALL'; const XHR = 'xhr'; @@ -83,7 +84,7 @@ export function renderDuration(r) { })) @connect(state => ({ timelinePointer: state.getIn(['sessions', 'timelinePointer']), -})) +}), { setTimelinePointer }) export default class Network extends React.PureComponent { state = { filter: '', @@ -96,6 +97,7 @@ export default class Network extends React.PureComponent { pause(); jump(e.time); this.setState({ currentIndex: index }) + this.props.setTimelinePointer(null); } onTabClick = activeTab => this.setState({ activeTab }) From 2aad7ac139847f2cac376acdc7904096f2bf1acb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 16:52:39 +0100 Subject: [PATCH 021/206] feat(DB): FOSS & EE match indexes --- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 41 ++++++++++++++++- .../db/init_dbs/postgresql/init_schema.sql | 45 ++++++++++++++----- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 3 ++ .../db/init_dbs/postgresql/init_schema.sql | 6 ++- 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index afbe2a218..2214c11a2 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -2,7 +2,6 @@ BEGIN; CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); --- FOS: CREATE INDEX IF NOT EXISTS pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; CREATE INDEX IF NOT EXISTS pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; CREATE INDEX IF NOT EXISTS pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; @@ -17,6 +16,46 @@ CREATE INDEX IF NOT EXISTS pages_timestamp_metgt0_idx ON events.pages (timestamp time_to_interactive > 0; CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; +CREATE INDEX IF NOT EXISTS issues_project_id_idx ON issues (project_id); + +CREATE INDEX IF NOT EXISTS errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; +CREATE INDEX IF NOT EXISTS errors_project_id_error_id_idx ON public.errors (project_id, error_id); +CREATE INDEX IF NOT EXISTS errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; + +CREATE INDEX IF NOT EXISTS sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; +CREATE INDEX IF NOT EXISTS sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; +CREATE INDEX IF NOT EXISTS sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; + +CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); +CREATE INDEX IF NOT EXISTS jobs_project_id_idx ON jobs (project_id); +CREATE INDEX IF NOT EXISTS errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); +CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); +CREATE INDEX IF NOT EXISTS errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); +CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); +CREATE INDEX ON events.resources (timestamp); +CREATE INDEX ON events.resources (success); +CREATE INDEX ON public.projects (project_key); +CREATE INDEX IF NOT EXISTS resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; +CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); +CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); +CREATE INDEX IF NOT EXISTS resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; +CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; +CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); +CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; +CREATE INDEX IF NOT EXISTS resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); + +DROP TRIGGER IF EXISTS on_insert_or_update ON projects; +CREATE TRIGGER on_insert_or_update + AFTER INSERT OR UPDATE + ON projects + FOR EACH ROW +EXECUTE PROCEDURE notify_project(); + +UPDATE tenants +SET name='' +WHERE name ISNULL; +ALTER TABLE tenants + ALTER COLUMN name SET NOT NULL; END; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 517029893..8f23b22de 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -114,7 +114,7 @@ $$ ( tenant_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, user_id text NOT NULL DEFAULT generate_api_key(20), - name text, + name text NOT NULL, api_key text UNIQUE default generate_api_key(20) not null, created_at timestamp without time zone NOT NULL DEFAULT (now() at time zone 'utc'), deleted_at timestamp without time zone NULL DEFAULT NULL, @@ -281,6 +281,11 @@ $$ ); CREATE INDEX ON public.projects (project_key); + CREATE TRIGGER on_insert_or_update + AFTER INSERT OR UPDATE + ON projects + FOR EACH ROW + EXECUTE PROCEDURE notify_project(); -- --- alerts.sql --- @@ -350,10 +355,10 @@ $$ CONSTRAINT notification_tenant_xor_user CHECK ( tenant_id NOTNULL AND user_id ISNULL OR tenant_id ISNULL AND user_id NOTNULL ) ); - CREATE INDEX notifications_user_id_index ON public.notifications (user_id); - CREATE INDEX notifications_tenant_id_index ON public.notifications (tenant_id); - CREATE INDEX notifications_created_at_index ON public.notifications (created_at DESC); - CREATE INDEX notifications_created_at_epoch_idx ON public.notifications (CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) DESC); + CREATE INDEX notifications_user_id_index ON notifications (user_id); + CREATE INDEX notifications_tenant_id_index ON notifications (tenant_id); + CREATE INDEX notifications_created_at_index ON notifications (created_at DESC); + CREATE INDEX notifications_created_at_epoch_idx ON notifications (CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) DESC); CREATE TABLE user_viewed_notifications ( @@ -464,6 +469,7 @@ $$ CREATE INDEX ON issues (issue_id, type); CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); + CREATE INDEX issues_project_id_idx ON issues (project_id); -- --- errors.sql --- @@ -482,12 +488,15 @@ $$ stacktrace jsonb, --to save the stacktrace and not query S3 another time stacktrace_parsed_at timestamp ); - CREATE INDEX errors_error_id_idx ON errors (error_id); CREATE INDEX ON errors (project_id, source); CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); CREATE INDEX errors_project_id_idx ON public.errors (project_id); CREATE INDEX errors_project_id_status_idx ON public.errors (project_id, status); + CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; + CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); + CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; + CREATE INDEX errors_error_id_idx ON errors (error_id); CREATE TABLE user_favorite_errors ( @@ -569,7 +578,6 @@ $$ CREATE INDEX ON sessions (project_id, metadata_8); CREATE INDEX ON sessions (project_id, metadata_9); CREATE INDEX ON sessions (project_id, metadata_10); --- CREATE INDEX ON sessions (rehydration_id); CREATE INDEX ON sessions (project_id, watchdogs_score DESC); CREATE INDEX platform_idx ON public.sessions (platform); @@ -591,6 +599,9 @@ $$ CREATE INDEX sessions_user_country_gin_idx ON public.sessions (project_id, user_country); CREATE INDEX ON sessions (project_id, user_country); CREATE INDEX ON sessions (project_id, user_browser); + CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; + CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; + CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; @@ -623,7 +634,7 @@ $$ session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, PRIMARY KEY (user_id, session_id) ); - + CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); -- --- assignments.sql --- @@ -808,6 +819,10 @@ $$ ); CREATE INDEX ON events.errors (session_id); CREATE INDEX ON events.errors (timestamp); + CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); + CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); + CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); + CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); CREATE INDEX errors_error_id_idx ON events.errors (error_id); @@ -857,18 +872,25 @@ $$ PRIMARY KEY (session_id, message_id) ); CREATE INDEX ON events.resources (session_id); - CREATE INDEX ON events.resources (timestamp); - CREATE INDEX ON events.resources (success); CREATE INDEX ON events.resources (status); CREATE INDEX ON events.resources (type); CREATE INDEX ON events.resources (duration) WHERE duration > 0; CREATE INDEX ON events.resources (url_host); + CREATE INDEX ON events.resources (timestamp); + CREATE INDEX ON events.resources (success); CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); CREATE INDEX resources_url_idx ON events.resources (url); CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); CREATE INDEX resources_url_hostpath_idx ON events.resources (url_hostpath); - + CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; + CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); + CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); + CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; + CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; + CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); + CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; + CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); CREATE TABLE events.performance ( @@ -922,6 +944,7 @@ $$ ); CREATE INDEX ON jobs (status); CREATE INDEX ON jobs (start_at); + CREATE INDEX jobs_project_id_idx ON jobs (project_id); CREATE TABLE traces diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 531f99590..f3ba07e48 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -3,4 +3,7 @@ BEGIN; CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); + +CREATE INDEX ON events.errors (timestamp); +CREATE INDEX ON public.projects (project_key); END; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 79e846b8d..e2e0af48f 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -108,7 +108,7 @@ $$ CREATE EXTENSION IF NOT EXISTS pgcrypto; -- --- accounts.sql --- - CREATE TABLE IF NOT EXISTS public.tenants + CREATE TABLE tenants ( tenant_id integer NOT NULL DEFAULT 1, user_id text NOT NULL DEFAULT generate_api_key(20), @@ -256,6 +256,7 @@ $$ }'::jsonb -- ?????? ); + CREATE INDEX ON public.projects (project_key); CREATE TRIGGER on_insert_or_update AFTER INSERT OR UPDATE ON projects @@ -780,6 +781,7 @@ $$ PRIMARY KEY (session_id, message_id) ); CREATE INDEX ON events.errors (session_id); + CREATE INDEX ON events.errors (timestamp); CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); @@ -836,6 +838,8 @@ $$ CREATE INDEX ON events.resources (type); CREATE INDEX ON events.resources (duration) WHERE duration > 0; CREATE INDEX ON events.resources (url_host); + CREATE INDEX ON events.resources (timestamp); + CREATE INDEX ON events.resources (success); CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); CREATE INDEX resources_url_idx ON events.resources (url); From 86a344cf10190d98954d865110ad47c06aba4d53 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 17:11:24 +0100 Subject: [PATCH 022/206] feat(DB): EE changed delta file --- ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 2214c11a2..71046764a 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -58,4 +58,4 @@ ALTER TABLE tenants ALTER COLUMN name SET NOT NULL; -END; \ No newline at end of file +COMMIT; \ No newline at end of file From 3147dad2507c990c712ac116b30caeda9083bfa4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 17:15:11 +0100 Subject: [PATCH 023/206] feat(DB): FOSS changed delta file --- scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index f3ba07e48..5f5b8f913 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -6,4 +6,6 @@ CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (sessi CREATE INDEX ON events.errors (timestamp); CREATE INDEX ON public.projects (project_key); -END; \ No newline at end of file + + +COMMIT; \ No newline at end of file From 517cce7241b44c4498720beb266e1918e17d3eeb Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 27 Dec 2021 21:48:41 +0530 Subject: [PATCH 024/206] change(ui) - no data msg --- .../components/Assist/components/SessionList/SessionList.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/components/Assist/components/SessionList/SessionList.tsx b/frontend/app/components/Assist/components/SessionList/SessionList.tsx index aaab06fca..f556a8f1d 100644 --- a/frontend/app/components/Assist/components/SessionList/SessionList.tsx +++ b/frontend/app/components/Assist/components/SessionList/SessionList.tsx @@ -18,8 +18,8 @@ function SessionList(props: Props) { return (
{ props.list.map(session => ) } From 6ee062192393a8983fac9dec31fdad9033bac8d7 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 27 Dec 2021 21:50:40 +0530 Subject: [PATCH 025/206] change(ui) - removed log --- frontend/app/components/Session_/Network/Network.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frontend/app/components/Session_/Network/Network.js b/frontend/app/components/Session_/Network/Network.js index 11c7b410c..150cadc59 100644 --- a/frontend/app/components/Session_/Network/Network.js +++ b/frontend/app/components/Session_/Network/Network.js @@ -110,11 +110,6 @@ export default class Network extends React.PureComponent { this.setState({ filter: value, filteredList: value ? filtered : resources, currentIndex: 0 }); } - // onFilterChange = (e, { value }) => this.setState({ filter: value }) - - componentDidUpdate() { - console.log('test') - } static getDerivedStateFromProps(nextProps, prevState) { const { filteredList } = prevState; From 2d1600a4d97099055254294473b87f5c07ffcc6a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 19:20:59 +0100 Subject: [PATCH 026/206] feat(api): fix isNot search feat(api): fix no custom for input feat(api): search by events_count --- api/chalicelib/core/sessions.py | 18 +++++++++++++----- api/schemas.py | 22 ++++++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 70858481c..509607ddc 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -192,7 +192,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f f.value = helper.values_for_operator(value=f.value, op=f.operator) f_k = f"f_value{i}" full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} - op = __get_sql_operator(f.operator) + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator is_not = False if __is_negation_operator(f.operator): is_not = True @@ -288,6 +289,13 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints.append( _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) # --------------------------------------------------------------------------- if len(data.events) > 0: @@ -341,7 +349,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where.append( _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, value_key=e_k)) - if len(event.custom) > 0: + if event.custom is not None and len(event.custom) > 0: event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, value_key=f"custom{i}")) full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} @@ -404,7 +412,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where.append( _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", event.value, value_key=e_k)) - if len(event.custom) > 0: + if event.custom is not None and len(event.custom) > 0: event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, value_key="custom{i}")) full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} @@ -484,7 +492,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f if event_index == 0 or or_events: event_where += ss_constraints if is_not: - if event_index == 0: + if event_index == 0 or or_events: events_query_from.append(f"""\ (SELECT session_id, @@ -498,7 +506,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f AND start_ts >= %(startDate)s AND start_ts <= %(endDate)s AND duration IS NOT NULL - ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + ) {"" if or_events else ("AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ """) else: events_query_from.append(f"""\ diff --git a/api/schemas.py b/api/schemas.py index 47966c854..8c31b1772 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -395,6 +395,7 @@ class FilterType(str, Enum): platform = "PLATFORM" metadata = "METADATA" issue = "ISSUE" + events_count = "EVENTS_COUNT" class SearchEventOperator(str, Enum): @@ -438,7 +439,7 @@ class IssueType(str, Enum): class _SessionSearchEventRaw(BaseModel): - custom: Optional[List[Union[str, int]]] = Field(None) + custom: Optional[List[Union[int, str]]] = Field(None) customOperator: Optional[MathOperator] = Field(None) key: Optional[str] = Field(None) value: Union[str, List[str]] = Field(...) @@ -468,20 +469,29 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): class _SessionSearchFilterSchema(BaseModel): custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) - value: Union[Optional[Union[str, int, IssueType, PlatformType]], - Optional[List[Union[str, int, IssueType, PlatformType]]]] = Field(...) + value: Union[Optional[Union[IssueType, PlatformType, int, str]], + Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...) type: FilterType = Field(...) - operator: SearchEventOperator = Field(...) + operator: Union[SearchEventOperator, MathOperator] = Field(...) source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) @root_validator def check_card_number_omitted(cls, values): if values.get("type") == FilterType.issue: for v in values.get("value"): - assert IssueType(v) + assert isinstance(v, IssueType), f"value should be of type IssueType for {values.get('type')} filter" elif values.get("type") == FilterType.platform: for v in values.get("value"): - assert PlatformType(v) + assert isinstance(v, PlatformType), \ + f"value should be of type PlatformType for {values.get('type')} filter" + elif values.get("type") == FilterType.events_count: + assert isinstance(values.get("operator"), MathOperator), \ + f"operator should be of type MathOperator for {values.get('type')} filter" + for v in values.get("value"): + assert isinstance(v, int), f"value should be of type int for {values.get('type')} filter" + else: + assert isinstance(values.get("operator"), SearchEventOperator), \ + f"operator should be of type SearchEventOperator for {values.get('type')} filter" return values From c49621314e437be6ea550c76e0811e62d73f28fb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 19:39:50 +0100 Subject: [PATCH 027/206] feat(api): search by TTFB --- api/chalicelib/core/performance_event.py | 1 + api/chalicelib/core/sessions.py | 6 ++++-- api/schemas.py | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py index d8cb2da6b..194ab1895 100644 --- a/api/chalicelib/core/performance_event.py +++ b/api/chalicelib/core/performance_event.py @@ -4,5 +4,6 @@ import schemas def get_col(perf: schemas.PerformanceEventType): return { schemas.PerformanceEventType.location_dom_complete: "dom_building_time", + schemas.PerformanceEventType.ttfb: "ttfb", # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" }.get(perf) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 509607ddc..cf8e93c1d 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -440,8 +440,9 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where.append( _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", event.value, value_key=e_k)) - elif event_type == [schemas.PerformanceEventType.location_dom_complete, - schemas.PerformanceEventType.location_largest_contentful_paint_time]: + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.ttfb]: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " if not is_any: event_where.append( @@ -488,6 +489,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: + print("LLLLLLLLLLLLLL") continue if event_index == 0 or or_events: event_where += ss_constraints diff --git a/api/schemas.py b/api/schemas.py index 8c31b1772..72eb4f65b 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -372,6 +372,7 @@ class PerformanceEventType(str, Enum): location_dom_complete = "DOM_COMPLETE" location_largest_contentful_paint_time = "LARGEST_CONTENTFUL_PAINT_TIME" time_between_events = "TIME_BETWEEN_EVENTS" + ttfb = "TTFB" class FilterType(str, Enum): @@ -459,6 +460,9 @@ class _SessionSearchEventRaw(BaseModel): assert isinstance(values["value"][0], _SessionSearchEventRaw) \ and isinstance(values["value"][1], _SessionSearchEventRaw) \ , f"event should be of type _SessionSearchEventRaw for {PerformanceEventType.time_between_events}" + else: + for c in values["custom"]: + assert isinstance(c, int), f"custom value should be of type int for {values.get('type')}" return values From 41327a86b86335bd3bd6e344f3ad2a5657e54220 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 27 Dec 2021 19:46:41 +0100 Subject: [PATCH 028/206] feat(api): numeric performance event not null and greater than 0 --- api/chalicelib/core/sessions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index cf8e93c1d..183bdd242 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -450,9 +450,10 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event.value, value_key=e_k)) e_k += "_custom" full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + colname=performance_event.get_col(event_type) event_where.append( _multiple_conditions( - f"main.{performance_event.get_col(event_type)} {event.customOperator} %({e_k})s", + f"main.{colname} {event.customOperator} %({e_k})s AND {colname} IS NOT NULL AND {colname}>0", event.custom, value_key=e_k)) elif event_type == schemas.PerformanceEventType.time_between_events: event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " From 00273a8e0e60e930636b911d2173dc6e2613fa0b Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Tue, 28 Dec 2021 01:26:01 +0300 Subject: [PATCH 029/206] Fix typo --- frontend/app/components/Session/LivePlayer.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/app/components/Session/LivePlayer.js b/frontend/app/components/Session/LivePlayer.js index 4007836ff..34c56721e 100644 --- a/frontend/app/components/Session/LivePlayer.js +++ b/frontend/app/components/Session/LivePlayer.js @@ -76,11 +76,8 @@ export default withRequest({ showAssist: state.getIn([ 'sessions', 'showChatWindow' ]), jwt: state.get('jwt'), fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), -<<<<<<< HEAD hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), -======= isEnterprise: state.getIn([ 'user', 'client', 'edition' ]) === 'ee', ->>>>>>> e2eec140e9dce5709f8ff9fd3b114a1e2a0f0e11 }), { toggleFullscreen, closeBottomBlock }, -)(WebPlayer))); \ No newline at end of file +)(WebPlayer))); From b00c557e25954ba3e0e9aab24d3d251d508b1ee3 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 28 Dec 2021 16:45:43 +0530 Subject: [PATCH 030/206] fix(ui) - redirect to destination path on login --- frontend/app/Router.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/frontend/app/Router.js b/frontend/app/Router.js index e7f05190e..06b71e3cb 100644 --- a/frontend/app/Router.js +++ b/frontend/app/Router.js @@ -76,6 +76,9 @@ const ONBOARDING_REDIRECT_PATH = routes.onboarding(OB_DEFAULT_TAB); fetchUserInfo, fetchTenants }) class Router extends React.Component { + state = { + destinationPath: null + } constructor(props) { super(props); if (props.isLoggedIn) { @@ -85,10 +88,22 @@ class Router extends React.Component { props.fetchTenants(); } - componentDidUpdate(prevProps) { + componentDidMount() { + const { isLoggedIn, location } = this.props; + if (!isLoggedIn) { + this.setState({ destinationPath: location.pathname }); + } + } + + componentDidUpdate(prevProps, prevState) { if (prevProps.email !== this.props.email && !this.props.email) { this.props.fetchTenants(); } + + if (!prevProps.isLoggedIn && this.props.isLoggedIn && this.state.destinationPath !== routes.login()) { + this.props.history.push(this.state.destinationPath); + this.setState({ destinationPath: null }); + } } render() { From 2bc486f744f8e6d32c7c6b3b83a6218a27fe8fd1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Dec 2021 12:54:39 +0100 Subject: [PATCH 031/206] feat(api): search sessions by AVG_CPU_LOAD --- api/chalicelib/core/performance_event.py | 5 +++-- api/chalicelib/core/sessions.py | 21 +++++++++++++++------ api/schemas.py | 3 ++- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py index 194ab1895..ad55fb2d9 100644 --- a/api/chalicelib/core/performance_event.py +++ b/api/chalicelib/core/performance_event.py @@ -3,7 +3,8 @@ import schemas def get_col(perf: schemas.PerformanceEventType): return { - schemas.PerformanceEventType.location_dom_complete: "dom_building_time", - schemas.PerformanceEventType.ttfb: "ttfb", + schemas.PerformanceEventType.location_dom_complete: {"column": "dom_building_time", "extraJoin": None}, + schemas.PerformanceEventType.location_ttfb: {"column": "ttfb", "extraJoin": None}, + schemas.PerformanceEventType.location_avg_cpu_load: {"column": "avg_cpu", "extraJoin": "events.performance"}, # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" }.get(perf) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 183bdd242..e086ab099 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -442,19 +442,28 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event.value, value_key=e_k)) elif event_type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, - schemas.PerformanceEventType.ttfb]: + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load + ]: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] if not is_any: event_where.append( _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", event.value, value_key=e_k)) e_k += "_custom" full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - colname=performance_event.get_col(event_type) - event_where.append( - _multiple_conditions( - f"main.{colname} {event.customOperator} %({e_k})s AND {colname} IS NOT NULL AND {colname}>0", - event.custom, value_key=e_k)) + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) elif event_type == schemas.PerformanceEventType.time_between_events: event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " if not isinstance(event.value[0].value, list): diff --git a/api/schemas.py b/api/schemas.py index 72eb4f65b..aabfad967 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -372,7 +372,8 @@ class PerformanceEventType(str, Enum): location_dom_complete = "DOM_COMPLETE" location_largest_contentful_paint_time = "LARGEST_CONTENTFUL_PAINT_TIME" time_between_events = "TIME_BETWEEN_EVENTS" - ttfb = "TTFB" + location_ttfb = "TTFB" + location_avg_cpu_load = "AVG_CPU_LOAD" class FilterType(str, Enum): From d227e7b3703b22f6c42c29e9a6aa17dc7a4e28d0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Dec 2021 16:44:15 +0100 Subject: [PATCH 032/206] feat(api): search sessions by AVG_MEMORY_USAGE --- api/chalicelib/core/performance_event.py | 2 ++ api/chalicelib/core/sessions.py | 3 ++- api/schemas.py | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py index ad55fb2d9..31ba69578 100644 --- a/api/chalicelib/core/performance_event.py +++ b/api/chalicelib/core/performance_event.py @@ -6,5 +6,7 @@ def get_col(perf: schemas.PerformanceEventType): schemas.PerformanceEventType.location_dom_complete: {"column": "dom_building_time", "extraJoin": None}, schemas.PerformanceEventType.location_ttfb: {"column": "ttfb", "extraJoin": None}, schemas.PerformanceEventType.location_avg_cpu_load: {"column": "avg_cpu", "extraJoin": "events.performance"}, + schemas.PerformanceEventType.location_avg_memory_usage: {"column": "avg_used_js_heap_size", + "extraJoin": "events.performance"}, # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" }.get(perf) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index e086ab099..d0c674007 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -443,7 +443,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f elif event_type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, schemas.PerformanceEventType.location_ttfb, - schemas.PerformanceEventType.location_avg_cpu_load + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage ]: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " col = performance_event.get_col(event_type) diff --git a/api/schemas.py b/api/schemas.py index aabfad967..de69f413c 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -374,6 +374,7 @@ class PerformanceEventType(str, Enum): time_between_events = "TIME_BETWEEN_EVENTS" location_ttfb = "TTFB" location_avg_cpu_load = "AVG_CPU_LOAD" + location_avg_memory_usage = "AVG_MEMORY_USAGE" class FilterType(str, Enum): From 93b7e4784ced59b74515a7b21b9b13e4d32c63e5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Dec 2021 20:20:09 +0100 Subject: [PATCH 033/206] feat(DB): UTM columns feat(api): refactored code feat(api): search sessions by UTM --- api/chalicelib/core/assist.py | 3 +- api/chalicelib/core/dashboard.py | 15 +- api/chalicelib/core/events.py | 22 +- api/chalicelib/core/insights.py | 21 +- api/chalicelib/core/sessions.py | 58 +++- api/chalicelib/core/sessions_metas.py | 265 ++++++++++-------- api/chalicelib/core/significance.py | 21 +- api/routers/core.py | 7 +- api/schemas.py | 3 + .../db/init_dbs/clickhouse/1.9.9/1.9.9.sql | 4 + .../init_dbs/clickhouse/create/sessions.sql | 48 ++-- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 8 + .../db/init_dbs/postgresql/init_schema.sql | 6 + 13 files changed, 284 insertions(+), 197 deletions(-) create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql diff --git a/api/chalicelib/core/assist.py b/api/chalicelib/core/assist.py index 8242e69ff..cd76d0be4 100644 --- a/api/chalicelib/core/assist.py +++ b/api/chalicelib/core/assist.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.utils import pg_client, helper from chalicelib.core import projects, sessions, sessions_metas import requests @@ -44,7 +45,7 @@ def get_live_sessions(project_id, filters=None): continue filter_type = f["type"].upper() f["value"] = sessions.__get_sql_value_multiple(f["value"]) - if filter_type == sessions_metas.meta_type.USERID: + if filter_type == schemas.FilterType.user_id: op = sessions.__get_sql_operator(f["operator"]) extra_constraints.append(f"user_id {op} %(value_{i})s") extra_params[f"value_{i}"] = helper.string_to_sql_like_with_op(f["value"][0], op) diff --git a/api/chalicelib/core/dashboard.py b/api/chalicelib/core/dashboard.py index 919d6aa5a..9cd88eb6a 100644 --- a/api/chalicelib/core/dashboard.py +++ b/api/chalicelib/core/dashboard.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.core import metadata from chalicelib.utils import args_transformer from chalicelib.utils import helper, dev @@ -94,25 +95,25 @@ def __get_meta_constraint(project_id, data): else: filter_type = f["key"].upper() filter_type = [filter_type, "USER" + filter_type, filter_type[4:]] - if any(item in [sessions_metas.meta_type.USERBROWSER] \ + if any(item in [schemas.FilterType.user_browser] \ for item in filter_type): constraints.append(f"sessions.user_browser = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS] \ + elif any(item in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios] \ for item in filter_type): constraints.append(f"sessions.user_os = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS] \ + elif any(item in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios] \ for item in filter_type): constraints.append(f"sessions.user_device = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS] \ + elif any(item in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios] \ for item in filter_type): constraints.append(f"sessions.user_country = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS] \ + elif any(item in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios] \ for item in filter_type): constraints.append(f"sessions.user_id = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.USERANONYMOUSID, sessions_metas.meta_type.USERANONYMOUSID_IOS] \ + elif any(item in [schemas.FilterType.user_anonymous_id, schemas.FilterType.user_anonymous_id_ios] \ for item in filter_type): constraints.append(f"sessions.user_anonymous_id = %({f['key']}_{i})s") - elif any(item in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS] \ + elif any(item in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios] \ for item in filter_type): constraints.append(f"sessions.rev_id = %({f['key']}_{i})s") return constraints diff --git a/api/chalicelib/core/events.py b/api/chalicelib/core/events.py index deb16cb8b..0a330d625 100644 --- a/api/chalicelib/core/events.py +++ b/api/chalicelib/core/events.py @@ -390,18 +390,18 @@ def search_pg2(text, event_type, project_id, source, key): if not event_type: return {"data": __get_autocomplete_table(text, project_id)} - if event_type.upper() in SUPPORTED_TYPES.keys(): - rows = SUPPORTED_TYPES[event_type.upper()].get(project_id=project_id, value=text, key=key, source=source) - if event_type.upper() + "_IOS" in SUPPORTED_TYPES.keys(): - rows += SUPPORTED_TYPES[event_type.upper() + "_IOS"].get(project_id=project_id, value=text, key=key, - source=source) - elif event_type.upper() + "_IOS" in SUPPORTED_TYPES.keys(): - rows = SUPPORTED_TYPES[event_type.upper() + "_IOS"].get(project_id=project_id, value=text, key=key, - source=source) - elif event_type.upper() in sessions_metas.SUPPORTED_TYPES.keys(): + if event_type in SUPPORTED_TYPES.keys(): + rows = SUPPORTED_TYPES[event_type].get(project_id=project_id, value=text, key=key, source=source) + if event_type + "_IOS" in SUPPORTED_TYPES.keys(): + rows += SUPPORTED_TYPES[event_type + "_IOS"].get(project_id=project_id, value=text, key=key, + source=source) + elif event_type + "_IOS" in SUPPORTED_TYPES.keys(): + rows = SUPPORTED_TYPES[event_type + "_IOS"].get(project_id=project_id, value=text, key=key, + source=source) + elif event_type in sessions_metas.SUPPORTED_TYPES.keys(): return sessions_metas.search(text, event_type, project_id) - elif event_type.upper().endswith("_IOS") \ - and event_type.upper()[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys(): + elif event_type.endswith("_IOS") \ + and event_type[:-len("_IOS")] in sessions_metas.SUPPORTED_TYPES.keys(): return sessions_metas.search(text, event_type, project_id) else: return {"errors": ["unsupported event"]} diff --git a/api/chalicelib/core/insights.py b/api/chalicelib/core/insights.py index 79b32a4b1..08adfd3ca 100644 --- a/api/chalicelib/core/insights.py +++ b/api/chalicelib/core/insights.py @@ -1,3 +1,4 @@ +import schemas from chalicelib.core import sessions_metas from chalicelib.utils import helper, dev from chalicelib.utils import pg_client @@ -45,7 +46,7 @@ def journey(project_id, startTimestamp=TimeUTC.now(delta_days=-1), endTimestamp= elif f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query_subset.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] @@ -300,7 +301,7 @@ def feature_retention(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -390,7 +391,7 @@ def feature_acquisition(project_id, startTimestamp=TimeUTC.now(delta_days=-70), elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -477,7 +478,7 @@ def feature_popularity_frequency(project_id, startTimestamp=TimeUTC.now(delta_da if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] @@ -543,7 +544,7 @@ def feature_adoption(project_id, startTimestamp=TimeUTC.now(delta_days=-70), end elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -613,7 +614,7 @@ def feature_adoption_top_users(project_id, startTimestamp=TimeUTC.now(delta_days elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -674,7 +675,7 @@ def feature_adoption_daily_usage(project_id, startTimestamp=TimeUTC.now(delta_da elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query_chart.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] @@ -737,7 +738,7 @@ def feature_intensity(project_id, startTimestamp=TimeUTC.now(delta_days=-70), en if f["type"] == "EVENT_TYPE" and JOURNEY_TYPES.get(f["value"]): event_table = JOURNEY_TYPES[f["value"]]["table"] event_column = JOURNEY_TYPES[f["value"]]["column"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] pg_sub_query.append(f"length({event_column})>2") @@ -772,7 +773,7 @@ def users_active(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTime for f in filters: if f["type"] == "PERIOD" and f["value"] in ["DAY", "WEEK"]: period = f["value"] - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query_chart.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] @@ -844,7 +845,7 @@ def users_slipping(project_id, startTimestamp=TimeUTC.now(delta_days=-70), endTi elif f["type"] == "EVENT_VALUE": event_value = f["value"] default = False - elif f["type"] in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif f["type"] in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: pg_sub_query.append(f"sessions.user_id = %(user_id)s") extra_values["user_id"] = f["value"] event_table = JOURNEY_TYPES[event_type]["table"] diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index d0c674007..0067b7c29 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1,5 +1,5 @@ import schemas -from chalicelib.core import events, sessions_metas, metadata, events_ios, \ +from chalicelib.core import events, metadata, events_ios, \ sessions_mobs, issues, projects, errors, resources, assist, performance_event from chalicelib.utils import pg_client, helper, dev @@ -194,37 +194,71 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} op = __get_sql_operator(f.operator) \ if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) is_not = False if __is_negation_operator(f.operator): is_not = True # op = __reverse_sql_operator(op) - if filter_type == sessions_metas.meta_type.USERBROWSER: + if filter_type == schemas.FilterType.user_browser: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: # op = __get_sql_operator_multiple(f.operator) extra_constraints.append( _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('s.utm_source IS NOT NULL') + ss_constraints.append('ms.utm_source IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('s.utm_medium IS NOT NULL') + ss_constraints.append('ms.utm_medium IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('s.utm_campaign IS NOT NULL') + ss_constraints.append('ms.utm_campaign IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.duration: if len(f.value) > 0 and f.value[0] is not None: extra_constraints.append("s.duration >= %(minDuration)s") @@ -234,7 +268,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f extra_constraints.append("s.duration <= %(maxDuration)s") ss_constraints.append("ms.duration <= %(maxDuration)s") full_args["maxDuration"] = f.value[1] - elif filter_type == sessions_metas.meta_type.REFERRER: + elif filter_type == schemas.FilterType.referrer: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" # op = __get_sql_operator_multiple(f.operator) @@ -253,14 +287,14 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints.append( _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: # op = __get_sql_operator(f.operator) extra_constraints.append( _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, - sessions_metas.meta_type.USERANONYMOUSID_IOS]: + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: # op = __get_sql_operator(f.operator) extra_constraints.append( _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, @@ -268,7 +302,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f ss_constraints.append( _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: # op = __get_sql_operator(f.operator) extra_constraints.append( _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) @@ -695,8 +729,8 @@ def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): available_keys = metadata.get_keys_by_projects(project_ids) for i in available_keys: - available_keys[i]["user_id"] = sessions_metas.meta_type.USERID - available_keys[i]["user_anonymous_id"] = sessions_metas.meta_type.USERANONYMOUSID + available_keys[i]["user_id"] = schemas.FilterType.user_id + available_keys[i]["user_anonymous_id"] = schemas.FilterType.user_anonymous_id results = {} for i in project_ids: if m_key not in available_keys[i].values(): diff --git a/api/chalicelib/core/sessions_metas.py b/api/chalicelib/core/sessions_metas.py index 1e55720a9..1d342d03f 100644 --- a/api/chalicelib/core/sessions_metas.py +++ b/api/chalicelib/core/sessions_metas.py @@ -9,40 +9,47 @@ def get_key_values(project_id): cur.mogrify( f"""\ SELECT ARRAY_AGG(DISTINCT s.user_os - ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='web') AS {meta_type.USEROS}, + ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='web') AS {schemas.FilterType.user_os}, ARRAY_AGG(DISTINCT s.user_browser ORDER BY s.user_browser) - FILTER ( WHERE s.user_browser IS NOT NULL AND s.platform='web') AS {meta_type.USERBROWSER}, + FILTER ( WHERE s.user_browser IS NOT NULL AND s.platform='web') AS {schemas.FilterType.user_browser}, ARRAY_AGG(DISTINCT s.user_device ORDER BY s.user_device) - FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='web') AS {meta_type.USERDEVICE}, + FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='web') AS {schemas.FilterType.user_device}, ARRAY_AGG(DISTINCT s.user_country ORDER BY s.user_country) - FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='web')::text[] AS {meta_type.USERCOUNTRY}, + FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='web')::text[] AS {schemas.FilterType.user_country}, ARRAY_AGG(DISTINCT s.user_id - ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='web') AS {meta_type.USERID}, + ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='web') AS {schemas.FilterType.user_id}, ARRAY_AGG(DISTINCT s.user_anonymous_id - ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='web') AS {meta_type.USERANONYMOUSID}, + ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='web') AS {schemas.FilterType.user_anonymous_id}, ARRAY_AGG(DISTINCT s.rev_id - ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='web') AS {meta_type.REVID}, + ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='web') AS {schemas.FilterType.rev_id}, ARRAY_AGG(DISTINCT p.referrer ORDER BY p.referrer) - FILTER ( WHERE p.referrer != '' ) AS {meta_type.REFERRER}, + FILTER ( WHERE p.referrer != '' ) AS {schemas.FilterType.referrer}, + + ARRAY_AGG(DISTINCT s.utm_source + ORDER BY s.utm_source) FILTER ( WHERE s.utm_source IS NOT NULL AND s.utm_source != 'none' AND s.utm_source != '') AS {schemas.FilterType.utm_source}, + ARRAY_AGG(DISTINCT s.utm_medium + ORDER BY s.utm_medium) FILTER ( WHERE s.utm_medium IS NOT NULL AND s.utm_medium != 'none' AND s.utm_medium != '') AS {schemas.FilterType.utm_medium}, + ARRAY_AGG(DISTINCT s.utm_campaign + ORDER BY s.utm_campaign) FILTER ( WHERE s.utm_campaign IS NOT NULL AND s.utm_campaign != 'none' AND s.utm_campaign != '') AS {schemas.FilterType.utm_campaign}, ARRAY_AGG(DISTINCT s.user_os - ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='ios' ) AS {meta_type.USEROS_IOS}, + ORDER BY s.user_os) FILTER ( WHERE s.user_os IS NOT NULL AND s.platform='ios' ) AS {schemas.FilterType.user_os_ios}, ARRAY_AGG(DISTINCT s.user_device ORDER BY s.user_device) - FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='ios') AS {meta_type.USERDEVICE}, + FILTER ( WHERE s.user_device IS NOT NULL AND s.user_device != '' AND s.platform='ios') AS {schemas.FilterType.user_device_ios}, ARRAY_AGG(DISTINCT s.user_country ORDER BY s.user_country) - FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='ios')::text[] AS {meta_type.USERCOUNTRY_IOS}, + FILTER ( WHERE s.user_country IS NOT NULL AND s.platform='ios')::text[] AS {schemas.FilterType.user_country_ios}, ARRAY_AGG(DISTINCT s.user_id - ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='ios') AS {meta_type.USERID_IOS}, + ORDER BY s.user_id) FILTER ( WHERE s.user_id IS NOT NULL AND s.user_id != 'none' AND s.user_id != '' AND s.platform='ios') AS {schemas.FilterType.user_id_ios}, ARRAY_AGG(DISTINCT s.user_anonymous_id - ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='ios') AS {meta_type.USERANONYMOUSID_IOS}, + ORDER BY s.user_anonymous_id) FILTER ( WHERE s.user_anonymous_id IS NOT NULL AND s.user_anonymous_id != 'none' AND s.user_anonymous_id != '' AND s.platform='ios') AS {schemas.FilterType.user_anonymous_id_ios}, ARRAY_AGG(DISTINCT s.rev_id - ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='ios') AS {meta_type.REVID_IOS} + ORDER BY s.rev_id) FILTER ( WHERE s.rev_id IS NOT NULL AND s.platform='ios') AS {schemas.FilterType.rev_id_ios} FROM public.sessions AS s LEFT JOIN events.pages AS p USING (session_id) WHERE s.project_id = %(site_id)s;""", @@ -109,119 +116,137 @@ def __generic_autocomplete(typename): return f -class meta_type: - USEROS = schemas.FilterType.user_os - USERBROWSER = schemas.FilterType.user_browser - USERDEVICE = schemas.FilterType.user_device - USERCOUNTRY = schemas.FilterType.user_country - USERID = schemas.FilterType.user_id - USERANONYMOUSID = schemas.FilterType.user_anonymous_id - REFERRER = schemas.FilterType.referrer - REVID = schemas.FilterType.rev_id - # IOS - USEROS_IOS = schemas.FilterType.user_os_ios - USERDEVICE_IOS = schemas.FilterType.user_device_ios - USERCOUNTRY_IOS = schemas.FilterType.user_country_ios - USERID_IOS = schemas.FilterType.user_id_ios - USERANONYMOUSID_IOS = schemas.FilterType.user_anonymous_id_ios - REVID_IOS = schemas.FilterType.rev_id_ios - - SUPPORTED_TYPES = { - meta_type.USEROS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USEROS), - query=__generic_query(typename=meta_type.USEROS), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.USERBROWSER: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERBROWSER), - query=__generic_query(typename=meta_type.USERBROWSER), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.USERDEVICE: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERDEVICE), - query=__generic_query(typename=meta_type.USERDEVICE), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.USERCOUNTRY: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERCOUNTRY), - query=__generic_query(typename=meta_type.USERCOUNTRY), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERID), - query=__generic_query(typename=meta_type.USERID), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERANONYMOUSID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERANONYMOUSID), - query=__generic_query(typename=meta_type.USERANONYMOUSID), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.REVID: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REVID), - query=__generic_query(typename=meta_type.REVID), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.REFERRER: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REFERRER), - query=__generic_query(typename=meta_type.REFERRER), - value_limit=5, - starts_with="/", - starts_limit=5, - ignore_if_starts_with=[]), + schemas.FilterType.user_os: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_os), + query=__generic_query(typename=schemas.FilterType.user_os), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_browser: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_browser), + query=__generic_query(typename=schemas.FilterType.user_browser), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_device: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_device), + query=__generic_query(typename=schemas.FilterType.user_device), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_country: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_country), + query=__generic_query(typename=schemas.FilterType.user_country), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_id: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_id), + query=__generic_query(typename=schemas.FilterType.user_id), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_anonymous_id: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id), + query=__generic_query(typename=schemas.FilterType.user_anonymous_id), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.rev_id: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.rev_id), + query=__generic_query(typename=schemas.FilterType.rev_id), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.referrer: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.referrer), + query=__generic_query(typename=schemas.FilterType.referrer), + value_limit=5, + starts_with="/", + starts_limit=5, + ignore_if_starts_with=[]), + schemas.FilterType.utm_campaign: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.utm_campaign), + query=__generic_query(typename=schemas.FilterType.utm_campaign), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.utm_medium: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.utm_medium), + query=__generic_query(typename=schemas.FilterType.utm_medium), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.utm_source: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.utm_source), + query=__generic_query(typename=schemas.FilterType.utm_source), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), # IOS - meta_type.USEROS_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USEROS_IOS), - query=__generic_query(typename=meta_type.USEROS_IOS), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), - meta_type.USERDEVICE_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERDEVICE_IOS), - query=__generic_query(typename=meta_type.USERDEVICE_IOS), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.USERCOUNTRY_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERCOUNTRY_IOS), - query=__generic_query(typename=meta_type.USERCOUNTRY_IOS), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERID_IOS), - query=__generic_query(typename=meta_type.USERID_IOS), - value_limit=2, - starts_with="", - starts_limit=2, - ignore_if_starts_with=["/"]), - meta_type.USERANONYMOUSID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.USERANONYMOUSID_IOS), - query=__generic_query(typename=meta_type.USERANONYMOUSID_IOS), - value_limit=3, - starts_with="", - starts_limit=3, - ignore_if_starts_with=["/"]), - meta_type.REVID_IOS: SupportedFilter(get=__generic_autocomplete(typename=meta_type.REVID_IOS), - query=__generic_query(typename=meta_type.REVID_IOS), - value_limit=0, - starts_with="", - starts_limit=0, - ignore_if_starts_with=["/"]), + schemas.FilterType.user_os_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_os_ios), + query=__generic_query(typename=schemas.FilterType.user_os_ios), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_device_ios: SupportedFilter( + get=__generic_autocomplete( + typename=schemas.FilterType.user_device_ios), + query=__generic_query(typename=schemas.FilterType.user_device_ios), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_country_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_country_ios), + query=__generic_query(typename=schemas.FilterType.user_country_ios), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_id_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_id_ios), + query=__generic_query(typename=schemas.FilterType.user_id_ios), + value_limit=2, + starts_with="", + starts_limit=2, + ignore_if_starts_with=["/"]), + schemas.FilterType.user_anonymous_id_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.user_anonymous_id_ios), + query=__generic_query(typename=schemas.FilterType.user_anonymous_id_ios), + value_limit=3, + starts_with="", + starts_limit=3, + ignore_if_starts_with=["/"]), + schemas.FilterType.rev_id_ios: SupportedFilter( + get=__generic_autocomplete(typename=schemas.FilterType.rev_id_ios), + query=__generic_query(typename=schemas.FilterType.rev_id_ios), + value_limit=0, + starts_with="", + starts_limit=0, + ignore_if_starts_with=["/"]), } def search(text, meta_type, project_id): rows = [] - if meta_type.upper() not in list(SUPPORTED_TYPES.keys()): + if meta_type not in list(SUPPORTED_TYPES.keys()): return {"errors": ["unsupported type"]} - rows += SUPPORTED_TYPES[meta_type.upper()].get(project_id=project_id, text=text) - if meta_type.upper() + "_IOS" in list(SUPPORTED_TYPES.keys()): - rows += SUPPORTED_TYPES[meta_type.upper() + "_IOS"].get(project_id=project_id, text=text) + rows += SUPPORTED_TYPES[meta_type].get(project_id=project_id, text=text) + if meta_type + "_IOS" in list(SUPPORTED_TYPES.keys()): + rows += SUPPORTED_TYPES[meta_type + "_IOS"].get(project_id=project_id, text=text) return {"data": rows} diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 3f991b895..9fc55065f 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -59,22 +59,22 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: values = {**values, **sessions._multiple_values(helper.values_for_operator(value=f["value"], op=f["operator"]), value_key=f_k)} - if filter_type == sessions_metas.meta_type.USERBROWSER: + if filter_type == schemas.FilterType.user_browser: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_browser {op} %({f_k})s', f["value"], value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USEROS, sessions_metas.meta_type.USEROS_IOS]: + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_os {op} %({f_k})s', f["value"], value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERDEVICE, sessions_metas.meta_type.USERDEVICE_IOS]: + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_device {op} %({f_k})s', f["value"], value_key=f_k)) - elif filter_type in [sessions_metas.meta_type.USERCOUNTRY, sessions_metas.meta_type.USERCOUNTRY_IOS]: + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: # op = sessions.__get_sql_operator_multiple(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_country {op} %({f_k})s', f["value"], value_key=f_k)) @@ -85,7 +85,7 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: if len(f["value"]) > 1 and f["value"][1] is not None and int(f["value"][1]) > 0: first_stage_extra_constraints.append('s.duration <= %(maxDuration)s') values["maxDuration"] = f["value"][1] - elif filter_type == sessions_metas.meta_type.REFERRER: + elif filter_type == schemas.FilterType.referrer: # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" filter_extra_from = [f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)"] # op = sessions.__get_sql_operator_multiple(f["operator"]) @@ -102,18 +102,18 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: f's.{metadata.index_to_colname(meta_keys[f["key"]])} {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [sessions_metas.meta_type.USERID, sessions_metas.meta_type.USERID_IOS]: + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [sessions_metas.meta_type.USERANONYMOUSID, - sessions_metas.meta_type.USERANONYMOUSID_IOS]: + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.user_anonymous_id {op} %({f_k})s', f["value"], value_key=f_k)) # values[f_k] = helper.string_to_sql_like_with_op(f["value"][0], op) - elif filter_type in [sessions_metas.meta_type.REVID, sessions_metas.meta_type.REVID_IOS]: + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: # op = sessions.__get_sql_operator(f["operator"]) first_stage_extra_constraints.append( sessions._multiple_conditions(f's.rev_id {op} %({f_k})s', f["value"], value_key=f_k)) @@ -561,7 +561,8 @@ def get_top_insights(filter_d, project_id): "dropDueToIssues": 0 }] - counts = sessions.search2_pg(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), project_id=project_id, user_id=None, count_only=True) + counts = sessions.search2_pg(data=schemas.SessionsSearchCountSchema.parse_obj(filter_d), project_id=project_id, + user_id=None, count_only=True) output[0]["sessionsCount"] = counts["countSessions"] output[0]["usersCount"] = counts["countUsers"] return output, 0 diff --git a/api/routers/core.py b/api/routers/core.py index 800a59765..90f6bf3e1 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -98,12 +98,11 @@ def comment_assignment(projectId: int, sessionId: int, issueId: str, data: schem @app.get('/{projectId}/events/search', tags=["events"]) -def events_search(projectId: int, q: str, type: str = None, key: str = None, source: str = None, - context: schemas.CurrentContext = Depends(OR_context)): +def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schemas.EventType] = None, key: str = None, + source: str = None, context: schemas.CurrentContext = Depends(OR_context)): if len(q) == 0: return {"data": []} - result = events.search_pg2(text=q, event_type=type, project_id=projectId, source=source, - key=key) + result = events.search_pg2(text=q, event_type=type, project_id=projectId, source=source, key=key) return result diff --git a/api/schemas.py b/api/schemas.py index de69f413c..37118bd1d 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -399,6 +399,9 @@ class FilterType(str, Enum): metadata = "METADATA" issue = "ISSUE" events_count = "EVENTS_COUNT" + utm_source = "UTM_SOURCE" + utm_medium = "UTM_MEDIUM" + utm_campaign = "UTM_CAMPAIGN" class SearchEventOperator(str, Enum): diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql new file mode 100644 index 000000000..d427688f5 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql @@ -0,0 +1,4 @@ +ALTER TABLE sessions + ADD COLUMN utm_source Nullable(String), + ADD COLUMN utm_medium Nullable(String), + ADD COLUMN utm_campaign Nullable(String); diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql index 77d430d85..54901bdc4 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql @@ -1,22 +1,26 @@ -CREATE TABLE sessions ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - duration UInt32, - pages_count UInt16, - events_count UInt16, - errors_count UInt16 -) ENGINE = ReplacingMergeTree( duration ) -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime, session_id) -TTL datetime + INTERVAL 1 MONTH; +CREATE TABLE sessions +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt32, + pages_count UInt16, + events_count UInt16, + errors_count UInt16, + utm_source Nullable(String), + utm_medium Nullable(String), + utm_campaign Nullable(String) +) ENGINE = ReplacingMergeTree(duration) + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime, session_id) + TTL datetime + INTERVAL 1 MONTH; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 5f5b8f913..bac24d114 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -7,5 +7,13 @@ CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (sessi CREATE INDEX ON events.errors (timestamp); CREATE INDEX ON public.projects (project_key); +ALTER TABLE sessions + ADD COLUMN utm_source text NULL DEFAULT NULL, + ADD COLUMN utm_medium text NULL DEFAULT NULL, + ADD COLUMN utm_campaign text NULL DEFAULT NULL; + +CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); +CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); +CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index e2e0af48f..516caa6ec 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -514,6 +514,9 @@ $$ watchdogs_score bigint NOT NULL DEFAULT 0, issue_score bigint NOT NULL DEFAULT 0, issue_types issue_type[] NOT NULL DEFAULT '{}'::issue_type[], + utm_source text NULL DEFAULT NULL, + utm_medium text NULL DEFAULT NULL, + utm_campaign text NULL DEFAULT NULL, metadata_1 text DEFAULT NULL, metadata_2 text DEFAULT NULL, metadata_3 text DEFAULT NULL, @@ -568,6 +571,9 @@ $$ CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; + CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); + CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); + CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); ALTER TABLE public.sessions ADD CONSTRAINT web_browser_constraint CHECK ( From 4209b5b1b9b87d7ddfec4038d7057311b208a1e8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 28 Dec 2021 20:20:32 +0100 Subject: [PATCH 034/206] feat(DB): UTM columns --- ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 8 ++++++++ ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 71046764a..641f298c1 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -57,5 +57,13 @@ WHERE name ISNULL; ALTER TABLE tenants ALTER COLUMN name SET NOT NULL; +ALTER TABLE sessions + ADD COLUMN utm_source text NULL DEFAULT NULL, + ADD COLUMN utm_medium text NULL DEFAULT NULL, + ADD COLUMN utm_campaign text NULL DEFAULT NULL; + +CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); +CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); +CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 8f23b22de..673906028 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -549,6 +549,9 @@ $$ watchdogs_score bigint NOT NULL DEFAULT 0, issue_score bigint NOT NULL DEFAULT 0, issue_types issue_type[] NOT NULL DEFAULT '{}'::issue_type[], + utm_source text NULL DEFAULT NULL, + utm_medium text NULL DEFAULT NULL, + utm_campaign text NULL DEFAULT NULL, metadata_1 text DEFAULT NULL, metadata_2 text DEFAULT NULL, metadata_3 text DEFAULT NULL, @@ -605,7 +608,9 @@ $$ CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; - + CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); + CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); + CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); ALTER TABLE public.sessions ADD CONSTRAINT web_browser_constraint CHECK ( From 3412bf1567b49328c9367341a3b1349b17a16d34 Mon Sep 17 00:00:00 2001 From: NoGameNoFun <9070825+nogamenofun98@users.noreply.github.com> Date: Wed, 29 Dec 2021 09:33:16 +0800 Subject: [PATCH 035/206] Changed how the _config in AssistManager construct - Changed URL().host to URL().hostname so it wont attached port while the config set again another port and causing duplicate port when connect using peerjs. - Changed the port in the config to check first if the URL() has port defined, then using the defined port first, so user can use custom port number for ws, when there is no custom port defined at the domainUrl, then it will then check https or not and append the correct port. --- .../app/player/MessageDistributor/managers/AssistManager.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 2ccea1ad5..fc762dcc7 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -147,12 +147,13 @@ export default class AssistManager { return; } this.setStatus(ConnectionStatus.Connecting) + const urlObject = new URL(window.ENV.API_EDP) import('peerjs').then(({ default: Peer }) => { const _config = { // @ts-ignore - host: new URL(window.ENV.API_EDP).host, + host: urlObject.hostname, path: '/assist', - port: location.protocol === 'https:' ? 443 : 80, + port: urlObject.port === "" ? location.protocol === 'https:' ? 443 : 80 : urlObject.port, } if (this.config) { From 66ce43ea083e68f71c195725ec22bcb94f779e25 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 29 Dec 2021 17:37:55 +0100 Subject: [PATCH 036/206] feat(api): fixed search TIME_BETWEEN_EVENTS feat(api): added = to MathOperator --- api/chalicelib/core/sessions.py | 5 ++--- api/schemas.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 0067b7c29..01d14c7a1 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -505,8 +505,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event.value[0].value = [event.value[0].value] if not isinstance(event.value[1].value, list): event.value[1].value = [event.value[1].value] - event.value[0].value = helper.values_for_operator(value=event.value[0].value, op=event.operator) - event.value[1].value = helper.values_for_operator(value=event.value[1].value, op=event.operator) + event.value[0].value = helper.values_for_operator(value=event.value[0].value, op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, op=event.value[0].operator) e_k1 = e_k + "_e1" e_k2 = e_k + "_e2" full_args = {**full_args, @@ -534,7 +534,6 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f else: - print("LLLLLLLLLLLLLL") continue if event_index == 0 or or_events: event_where += ss_constraints diff --git a/api/schemas.py b/api/schemas.py index 37118bd1d..e3d7a5be4 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -313,6 +313,7 @@ class AlertColumn(str, Enum): class MathOperator(str, Enum): + _equal = "=" _less = "<" _greater = ">" _less_eq = "<=" From dcb69816e49a04ec09051c3e180f29b1230d35fd Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 31 Dec 2021 14:13:13 +0100 Subject: [PATCH 037/206] feat(api): FAILED_FETCH search feat(DB): new indexes --- api/chalicelib/core/performance_event.py | 2 ++ api/chalicelib/core/sessions.py | 29 +++++++++++++++++-- api/schemas.py | 4 +++ .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 7 +++-- .../db/init_dbs/postgresql/init_schema.sql | 1 + .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 7 +++-- .../db/init_dbs/postgresql/init_schema.sql | 2 +- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py index 31ba69578..98ac15b08 100644 --- a/api/chalicelib/core/performance_event.py +++ b/api/chalicelib/core/performance_event.py @@ -8,5 +8,7 @@ def get_col(perf: schemas.PerformanceEventType): schemas.PerformanceEventType.location_avg_cpu_load: {"column": "avg_cpu", "extraJoin": "events.performance"}, schemas.PerformanceEventType.location_avg_memory_usage: {"column": "avg_used_js_heap_size", "extraJoin": "events.performance"}, + schemas.PerformanceEventType.fetch_failed: {"column": "success", "extraJoin": None}, + schemas.PerformanceEventType.fetch_duration: {"column": "duration", "extraJoin": None}, # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" }.get(perf) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 01d14c7a1..46b32dfb6 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -474,6 +474,29 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event_where.append( _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", event.value, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + elif event_type == schemas.PerformanceEventType.fetch_duration: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) elif event_type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, schemas.PerformanceEventType.location_ttfb, @@ -505,8 +528,10 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f event.value[0].value = [event.value[0].value] if not isinstance(event.value[1].value, list): event.value[1].value = [event.value[1].value] - event.value[0].value = helper.values_for_operator(value=event.value[0].value, op=event.value[0].operator) - event.value[1].value = helper.values_for_operator(value=event.value[1].value, op=event.value[0].operator) + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) e_k1 = e_k + "_e1" e_k2 = e_k + "_e2" full_args = {**full_args, diff --git a/api/schemas.py b/api/schemas.py index e3d7a5be4..1d18c951b 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -376,6 +376,8 @@ class PerformanceEventType(str, Enum): location_ttfb = "TTFB" location_avg_cpu_load = "AVG_CPU_LOAD" location_avg_memory_usage = "AVG_MEMORY_USAGE" + fetch_failed = "FETCH_FAILED" + fetch_duration = "FETCH_DURATION" class FilterType(str, Enum): @@ -457,6 +459,8 @@ class _SessionSearchEventRaw(BaseModel): @root_validator def check_card_number_omitted(cls, values): if isinstance(values.get("type"), PerformanceEventType): + if values.get("type") == PerformanceEventType.fetch_failed: + return values assert values.get("custom") is not None, "custom should not be None for PerformanceEventType" assert values.get("customOperator") is not None \ , "customOperator should not be None for PerformanceEventType" diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 641f298c1..cdb254b5e 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -62,8 +62,9 @@ ALTER TABLE sessions ADD COLUMN utm_medium text NULL DEFAULT NULL, ADD COLUMN utm_campaign text NULL DEFAULT NULL; -CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); -CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); -CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); +CREATE INDEX IF NOT EXISTS sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); +CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); +CREATE INDEX IF NOT EXISTS sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); +CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 673906028..8082e66b8 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -709,6 +709,7 @@ $$ THEN 8 ELSE 0 END)) gin_trgm_ops); + CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; -- --- events.sql --- CREATE SCHEMA IF NOT EXISTS events; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index bac24d114..6788f1b96 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -12,8 +12,9 @@ ALTER TABLE sessions ADD COLUMN utm_medium text NULL DEFAULT NULL, ADD COLUMN utm_campaign text NULL DEFAULT NULL; -CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); -CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); -CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); +CREATE INDEX IF NOT EXISTS sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); +CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); +CREATE INDEX IF NOT EXISTS sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); +CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 516caa6ec..60d8f2e70 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -672,7 +672,7 @@ $$ THEN 8 ELSE 0 END)) gin_trgm_ops); - + CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; -- --- events.sql --- CREATE SCHEMA IF NOT EXISTS events; From 6cc82b667670146a0dedfe695488656e550e27da Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 31 Dec 2021 15:39:57 +0100 Subject: [PATCH 038/206] feat(DB): indexes cleanup --- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 3 ++ .../db/init_dbs/postgresql/init_schema.sql | 38 +++++++++---------- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 4 ++ .../db/init_dbs/postgresql/init_schema.sql | 38 +++++++++---------- 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index cdb254b5e..96ad91371 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -67,4 +67,7 @@ CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING CREATE INDEX IF NOT EXISTS sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; +DROP INDEX IF EXISTS sessions_project_id_user_browser_idx1; +DROP INDEX IF EXISTS sessions_project_id_user_country_idx1; +ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 8082e66b8..14abde03f 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -565,24 +565,24 @@ $$ -- , -- rehydration_id integer REFERENCES rehydrations(rehydration_id) ON DELETE SET NULL ); - CREATE INDEX ON sessions (project_id, start_ts); - CREATE INDEX ON sessions (project_id, user_id); - CREATE INDEX ON sessions (project_id, user_anonymous_id); - CREATE INDEX ON sessions (project_id, user_device); - CREATE INDEX ON sessions (project_id, user_country); - CREATE INDEX ON sessions (project_id, user_browser); - CREATE INDEX ON sessions (project_id, metadata_1); - CREATE INDEX ON sessions (project_id, metadata_2); - CREATE INDEX ON sessions (project_id, metadata_3); - CREATE INDEX ON sessions (project_id, metadata_4); - CREATE INDEX ON sessions (project_id, metadata_5); - CREATE INDEX ON sessions (project_id, metadata_6); - CREATE INDEX ON sessions (project_id, metadata_7); - CREATE INDEX ON sessions (project_id, metadata_8); - CREATE INDEX ON sessions (project_id, metadata_9); - CREATE INDEX ON sessions (project_id, metadata_10); - CREATE INDEX ON sessions (project_id, watchdogs_score DESC); - CREATE INDEX platform_idx ON public.sessions (platform); + CREATE INDEX sessions_project_id_start_ts_idx ON sessions (project_id, start_ts); + CREATE INDEX sessions_project_id_user_id_idx ON sessions (project_id, user_id); + CREATE INDEX sessions_project_id_user_anonymous_id_idx ON sessions (project_id, user_anonymous_id); + CREATE INDEX sessions_project_id_user_device_idx ON sessions (project_id, user_device); + CREATE INDEX sessions_project_id_user_country_idx ON sessions (project_id, user_country); + CREATE INDEX sessions_project_id_user_browser_idx ON sessions (project_id, user_browser); + CREATE INDEX sessions_project_id_metadata_1_idx ON sessions (project_id, metadata_1); + CREATE INDEX sessions_project_id_metadata_2_idx ON sessions (project_id, metadata_2); + CREATE INDEX sessions_project_id_metadata_3_idx ON sessions (project_id, metadata_3); + CREATE INDEX sessions_project_id_metadata_4_idx ON sessions (project_id, metadata_4); + CREATE INDEX sessions_project_id_metadata_5_idx ON sessions (project_id, metadata_5); + CREATE INDEX sessions_project_id_metadata_6_idx ON sessions (project_id, metadata_6); + CREATE INDEX sessions_project_id_metadata_7_idx ON sessions (project_id, metadata_7); + CREATE INDEX sessions_project_id_metadata_8_idx ON sessions (project_id, metadata_8); + CREATE INDEX sessions_project_id_metadata_9_idx ON sessions (project_id, metadata_9); + CREATE INDEX sessions_project_id_metadata_10_idx ON sessions (project_id, metadata_10); + CREATE INDEX sessions_project_id_watchdogs_score_idx ON sessions (project_id, watchdogs_score DESC); + CREATE INDEX sessions_platform_idx ON public.sessions (platform); CREATE INDEX sessions_metadata1_gin_idx ON public.sessions USING GIN (metadata_1 gin_trgm_ops); CREATE INDEX sessions_metadata2_gin_idx ON public.sessions USING GIN (metadata_2 gin_trgm_ops); @@ -600,8 +600,6 @@ $$ CREATE INDEX sessions_user_id_gin_idx ON public.sessions USING GIN (user_id gin_trgm_ops); CREATE INDEX sessions_user_anonymous_id_gin_idx ON public.sessions USING GIN (user_anonymous_id gin_trgm_ops); CREATE INDEX sessions_user_country_gin_idx ON public.sessions (project_id, user_country); - CREATE INDEX ON sessions (project_id, user_country); - CREATE INDEX ON sessions (project_id, user_browser); CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 6788f1b96..8bc80177d 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -17,4 +17,8 @@ CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING CREATE INDEX IF NOT EXISTS sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + +DROP INDEX IF EXISTS sessions_project_id_user_browser_idx1; +DROP INDEX IF EXISTS sessions_project_id_user_country_idx1; +ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 60d8f2e70..eb9cd9312 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -528,24 +528,24 @@ $$ metadata_9 text DEFAULT NULL, metadata_10 text DEFAULT NULL ); - CREATE INDEX ON sessions (project_id, start_ts); - CREATE INDEX ON sessions (project_id, user_id); - CREATE INDEX ON sessions (project_id, user_anonymous_id); - CREATE INDEX ON sessions (project_id, user_device); - CREATE INDEX ON sessions (project_id, user_country); - CREATE INDEX ON sessions (project_id, user_browser); - CREATE INDEX ON sessions (project_id, metadata_1); - CREATE INDEX ON sessions (project_id, metadata_2); - CREATE INDEX ON sessions (project_id, metadata_3); - CREATE INDEX ON sessions (project_id, metadata_4); - CREATE INDEX ON sessions (project_id, metadata_5); - CREATE INDEX ON sessions (project_id, metadata_6); - CREATE INDEX ON sessions (project_id, metadata_7); - CREATE INDEX ON sessions (project_id, metadata_8); - CREATE INDEX ON sessions (project_id, metadata_9); - CREATE INDEX ON sessions (project_id, metadata_10); - CREATE INDEX ON sessions (project_id, watchdogs_score DESC); - CREATE INDEX platform_idx ON public.sessions (platform); + CREATE INDEX sessions_project_id_start_ts_idx ON sessions (project_id, start_ts); + CREATE INDEX sessions_project_id_user_id_idx ON sessions (project_id, user_id); + CREATE INDEX sessions_project_id_user_anonymous_id_idx ON sessions (project_id, user_anonymous_id); + CREATE INDEX sessions_project_id_user_device_idx ON sessions (project_id, user_device); + CREATE INDEX sessions_project_id_user_country_idx ON sessions (project_id, user_country); + CREATE INDEX sessions_project_id_user_browser_idx ON sessions (project_id, user_browser); + CREATE INDEX sessions_project_id_metadata_1_idx ON sessions (project_id, metadata_1); + CREATE INDEX sessions_project_id_metadata_2_idx ON sessions (project_id, metadata_2); + CREATE INDEX sessions_project_id_metadata_3_idx ON sessions (project_id, metadata_3); + CREATE INDEX sessions_project_id_metadata_4_idx ON sessions (project_id, metadata_4); + CREATE INDEX sessions_project_id_metadata_5_idx ON sessions (project_id, metadata_5); + CREATE INDEX sessions_project_id_metadata_6_idx ON sessions (project_id, metadata_6); + CREATE INDEX sessions_project_id_metadata_7_idx ON sessions (project_id, metadata_7); + CREATE INDEX sessions_project_id_metadata_8_idx ON sessions (project_id, metadata_8); + CREATE INDEX sessions_project_id_metadata_9_idx ON sessions (project_id, metadata_9); + CREATE INDEX sessions_project_id_metadata_10_idx ON sessions (project_id, metadata_10); + CREATE INDEX sessions_project_id_watchdogs_score_idx ON sessions (project_id, watchdogs_score DESC); + CREATE INDEX sessions_platform_idx ON public.sessions (platform); CREATE INDEX sessions_metadata1_gin_idx ON public.sessions USING GIN (metadata_1 gin_trgm_ops); CREATE INDEX sessions_metadata2_gin_idx ON public.sessions USING GIN (metadata_2 gin_trgm_ops); @@ -563,8 +563,6 @@ $$ CREATE INDEX sessions_user_id_gin_idx ON public.sessions USING GIN (user_id gin_trgm_ops); CREATE INDEX sessions_user_anonymous_id_gin_idx ON public.sessions USING GIN (user_anonymous_id gin_trgm_ops); CREATE INDEX sessions_user_country_gin_idx ON public.sessions (project_id, user_country); - CREATE INDEX ON sessions (project_id, user_country); - CREATE INDEX ON sessions (project_id, user_browser); CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; From 6fc3840fa0f06f9f63a8f9288cf3dcc6081faa2b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 31 Dec 2021 18:36:27 +0100 Subject: [PATCH 039/206] feat(DB): indexes cleanup feat(api): disable fetch_duration filter --- api/chalicelib/core/performance_event.py | 2 +- api/chalicelib/core/sessions.py | 28 ++++---- api/schemas.py | 2 +- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 2 + .../db/init_dbs/postgresql/init_schema.sql | 70 +++++++++---------- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 2 + .../db/init_dbs/postgresql/init_schema.sql | 70 +++++++++---------- 7 files changed, 90 insertions(+), 86 deletions(-) diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py index 98ac15b08..1311614d9 100644 --- a/api/chalicelib/core/performance_event.py +++ b/api/chalicelib/core/performance_event.py @@ -9,6 +9,6 @@ def get_col(perf: schemas.PerformanceEventType): schemas.PerformanceEventType.location_avg_memory_usage: {"column": "avg_used_js_heap_size", "extraJoin": "events.performance"}, schemas.PerformanceEventType.fetch_failed: {"column": "success", "extraJoin": None}, - schemas.PerformanceEventType.fetch_duration: {"column": "duration", "extraJoin": None}, + # schemas.PerformanceEventType.fetch_duration: {"column": "duration", "extraJoin": None}, # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" }.get(perf) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 46b32dfb6..8517b1161 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -483,20 +483,20 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f col = performance_event.get_col(event_type) colname = col["column"] event_where.append(f"main.{colname} = FALSE") - elif event_type == schemas.PerformanceEventType.fetch_duration: - event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - event.value, value_key=e_k)) - col = performance_event.get_col(event_type) - colname = col["column"] - tname = "main" - e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + # event.custom, value_key=e_k)) elif event_type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, schemas.PerformanceEventType.location_ttfb, diff --git a/api/schemas.py b/api/schemas.py index 1d18c951b..dc6ff5531 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -377,7 +377,7 @@ class PerformanceEventType(str, Enum): location_avg_cpu_load = "AVG_CPU_LOAD" location_avg_memory_usage = "AVG_MEMORY_USAGE" fetch_failed = "FETCH_FAILED" - fetch_duration = "FETCH_DURATION" + # fetch_duration = "FETCH_DURATION" class FilterType(str, Enum): diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 96ad91371..39fa107da 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -70,4 +70,6 @@ CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_co DROP INDEX IF EXISTS sessions_project_id_user_browser_idx1; DROP INDEX IF EXISTS sessions_project_id_user_country_idx1; ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; +ALTER INDEX IF EXISTS events.resources_duration_idx RENAME TO resources_duration_durationgt0_idx; +DROP INDEX IF EXISTS projects_project_key_idx1; COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 14abde03f..17d6c09c4 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -280,7 +280,7 @@ $$ }'::jsonb -- ?????? ); - CREATE INDEX ON public.projects (project_key); + CREATE INDEX projects_project_key_idx ON public.projects (project_key); CREATE TRIGGER on_insert_or_update AFTER INSERT OR UPDATE ON projects @@ -381,7 +381,7 @@ $$ is_public boolean NOT NULL DEFAULT False ); - CREATE INDEX ON public.funnels (user_id, is_public); + CREATE INDEX funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); -- --- announcements.sql --- @@ -466,7 +466,7 @@ $$ context_string text NOT NULL, context jsonb DEFAULT NULL ); - CREATE INDEX ON issues (issue_id, type); + CREATE INDEX issues_issue_id_type_idx ON issues (issue_id, type); CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); CREATE INDEX issues_project_id_idx ON issues (project_id); @@ -488,7 +488,7 @@ $$ stacktrace jsonb, --to save the stacktrace and not query S3 another time stacktrace_parsed_at timestamp ); - CREATE INDEX ON errors (project_id, source); + CREATE INDEX errors_project_id_source_idx ON errors (project_id, source); CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); CREATE INDEX errors_project_id_idx ON public.errors (project_id); @@ -650,7 +650,7 @@ $$ created_at timestamp default timezone('utc'::text, now()) NOT NULL, provider_data jsonb default '{}'::jsonb NOT NULL ); - CREATE INDEX ON assigned_sessions (session_id); + CREATE INDEX assigned_sessions_session_id_idx ON assigned_sessions (session_id); -- --- events_common.sql --- @@ -668,9 +668,9 @@ $$ level events_common.custom_level NOT NULL DEFAULT 'info', PRIMARY KEY (session_id, timestamp, seq_index) ); - CREATE INDEX ON events_common.customs (name); + CREATE INDEX customs_name_idx ON events_common.customs (name); CREATE INDEX customs_name_gin_idx ON events_common.customs USING GIN (name gin_trgm_ops); - CREATE INDEX ON events_common.customs (timestamp); + CREATE INDEX customs_timestamp_idx ON events_common.customs (timestamp); CREATE TABLE events_common.issues @@ -696,10 +696,10 @@ $$ success boolean NOT NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); - CREATE INDEX ON events_common.requests (url); - CREATE INDEX ON events_common.requests (duration); + CREATE INDEX requests_url_idx ON events_common.requests (url); + CREATE INDEX requests_duration_idx ON events_common.requests (duration); CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); - CREATE INDEX ON events_common.requests (timestamp); + CREATE INDEX requests_timestamp_idx ON events_common.requests (timestamp); CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE WHEN url LIKE 'http://%' THEN 7 @@ -735,10 +735,10 @@ $$ ttfb integer DEFAULT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.pages (session_id); + CREATE INDEX pages_session_id_idx ON events.pages (session_id); CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); - CREATE INDEX ON events.pages (timestamp); + CREATE INDEX pages_timestamp_idx ON events.pages (timestamp); CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); CREATE INDEX pages_base_path_idx ON events.pages (base_path); @@ -752,8 +752,8 @@ $$ THEN 8 ELSE 0 END)) gin_trgm_ops); - CREATE INDEX ON events.pages (response_time); - CREATE INDEX ON events.pages (response_end); + CREATE INDEX pages_response_time_idx ON events.pages (response_time); + CREATE INDEX pages_response_end_idx ON events.pages (response_end); CREATE INDEX pages_path_gin_idx ON events.pages USING GIN (path gin_trgm_ops); CREATE INDEX pages_path_idx ON events.pages (path); CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; @@ -786,10 +786,10 @@ $$ selector text DEFAULT '' NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.clicks (session_id); - CREATE INDEX ON events.clicks (label); + CREATE INDEX clicks_session_id_idx ON events.clicks (session_id); + CREATE INDEX clicks_label_idx ON events.clicks (label); CREATE INDEX clicks_label_gin_idx ON events.clicks USING GIN (label gin_trgm_ops); - CREATE INDEX ON events.clicks (timestamp); + CREATE INDEX clicks_timestamp_idx ON events.clicks (timestamp); CREATE INDEX clicks_label_session_id_timestamp_idx ON events.clicks (label, session_id, timestamp); CREATE INDEX clicks_url_idx ON events.clicks (url); CREATE INDEX clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops); @@ -806,11 +806,11 @@ $$ value text DEFAULT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.inputs (session_id); - CREATE INDEX ON events.inputs (label, value); + CREATE INDEX inputs_session_id_idx ON events.inputs (session_id); + CREATE INDEX inputs_label_idx ON events.inputs (label, value); CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); CREATE INDEX inputs_label_idx ON events.inputs (label); - CREATE INDEX ON events.inputs (timestamp); + CREATE INDEX inputs_timestamp_idx ON events.inputs (timestamp); CREATE INDEX inputs_label_session_id_timestamp_idx ON events.inputs (label, session_id, timestamp); CREATE TABLE events.errors @@ -821,8 +821,8 @@ $$ error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.errors (session_id); - CREATE INDEX ON events.errors (timestamp); + CREATE INDEX errors_session_id_idx ON events.errors (session_id); + CREATE INDEX errors_timestamp_idx ON events.errors (timestamp); CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); @@ -838,9 +838,9 @@ $$ name text NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.graphql (name); + CREATE INDEX graphql_name_idx ON events.graphql (name); CREATE INDEX graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); - CREATE INDEX ON events.graphql (timestamp); + CREATE INDEX graphql_timestamp_idx ON events.graphql (timestamp); CREATE TABLE events.state_actions ( @@ -850,9 +850,9 @@ $$ name text NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.state_actions (name); + CREATE INDEX state_actions_name_idx ON events.state_actions (name); CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); - CREATE INDEX ON events.state_actions (timestamp); + CREATE INDEX state_actions_timestamp_idx ON events.state_actions (timestamp); CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); @@ -875,13 +875,13 @@ $$ decoded_body_size integer NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.resources (session_id); - CREATE INDEX ON events.resources (status); - CREATE INDEX ON events.resources (type); - CREATE INDEX ON events.resources (duration) WHERE duration > 0; - CREATE INDEX ON events.resources (url_host); - CREATE INDEX ON events.resources (timestamp); - CREATE INDEX ON events.resources (success); + CREATE INDEX resources_session_id_idx ON events.resources (session_id); + CREATE INDEX resources_status_idx ON events.resources (status); + CREATE INDEX resources_type_idx ON events.resources (type); + CREATE INDEX resources_duration_durationgt0_idx ON events.resources (duration) WHERE duration > 0; + CREATE INDEX resources_url_host_idx ON events.resources (url_host); + CREATE INDEX resources_timestamp_idx ON events.resources (timestamp); + CREATE INDEX resources_success_idx ON events.resources (success); CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); CREATE INDEX resources_url_idx ON events.resources (url); @@ -946,8 +946,8 @@ $$ start_at timestamp NOT NULL, errors text NULL ); - CREATE INDEX ON jobs (status); - CREATE INDEX ON jobs (start_at); + CREATE INDEX jobs_status_idx ON jobs (status); + CREATE INDEX jobs_start_at_idx ON jobs (start_at); CREATE INDEX jobs_project_id_idx ON jobs (project_id); diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 8bc80177d..eb691bfbf 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -21,4 +21,6 @@ CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_co DROP INDEX IF EXISTS sessions_project_id_user_browser_idx1; DROP INDEX IF EXISTS sessions_project_id_user_country_idx1; ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; +ALTER INDEX IF EXISTS events.resources_duration_idx RENAME TO resources_duration_durationgt0_idx; +DROP INDEX IF EXISTS projects_project_key_idx1; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index eb9cd9312..793e53699 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -256,7 +256,7 @@ $$ }'::jsonb -- ?????? ); - CREATE INDEX ON public.projects (project_key); + CREATE INDEX projects_project_key_idx ON public.projects (project_key); CREATE TRIGGER on_insert_or_update AFTER INSERT OR UPDATE ON projects @@ -348,7 +348,7 @@ $$ is_public boolean NOT NULL DEFAULT False ); - CREATE INDEX ON public.funnels (user_id, is_public); + CREATE INDEX funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); -- --- announcements.sql --- @@ -432,7 +432,7 @@ $$ context_string text NOT NULL, context jsonb DEFAULT NULL ); - CREATE INDEX ON issues (issue_id, type); + CREATE INDEX issues_issue_id_type_idx ON issues (issue_id, type); CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); CREATE INDEX issues_project_id_idx ON issues (project_id); @@ -453,7 +453,7 @@ $$ stacktrace jsonb, --to save the stacktrace and not query S3 another time stacktrace_parsed_at timestamp ); - CREATE INDEX ON errors (project_id, source); + CREATE INDEX errors_project_id_source_idx ON errors (project_id, source); CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); CREATE INDEX errors_project_id_idx ON public.errors (project_id); @@ -613,7 +613,7 @@ $$ created_at timestamp default timezone('utc'::text, now()) NOT NULL, provider_data jsonb default '{}'::jsonb NOT NULL ); - CREATE INDEX ON assigned_sessions (session_id); + CREATE INDEX assigned_sessions_session_id_idx ON assigned_sessions (session_id); -- --- events_common.sql --- @@ -631,9 +631,9 @@ $$ level events_common.custom_level NOT NULL DEFAULT 'info', PRIMARY KEY (session_id, timestamp, seq_index) ); - CREATE INDEX ON events_common.customs (name); + CREATE INDEX customs_name_idx ON events_common.customs (name); CREATE INDEX customs_name_gin_idx ON events_common.customs USING GIN (name gin_trgm_ops); - CREATE INDEX ON events_common.customs (timestamp); + CREATE INDEX customs_timestamp_idx ON events_common.customs (timestamp); CREATE TABLE events_common.issues @@ -659,10 +659,10 @@ $$ success boolean NOT NULL, PRIMARY KEY (session_id, timestamp, seq_index) ); - CREATE INDEX ON events_common.requests (url); - CREATE INDEX ON events_common.requests (duration); + CREATE INDEX requests_url_idx ON events_common.requests (url); + CREATE INDEX requests_duration_idx ON events_common.requests (duration); CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); - CREATE INDEX ON events_common.requests (timestamp); + CREATE INDEX requests_timestamp_idx ON events_common.requests (timestamp); CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE WHEN url LIKE 'http://%' THEN 7 @@ -697,10 +697,10 @@ $$ ttfb integer DEFAULT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.pages (session_id); + CREATE INDEX pages_session_id_idx ON events.pages (session_id); CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); - CREATE INDEX ON events.pages (timestamp); + CREATE INDEX pages_timestamp_idx ON events.pages (timestamp); CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); CREATE INDEX pages_base_path_idx ON events.pages (base_path); @@ -714,8 +714,8 @@ $$ THEN 8 ELSE 0 END)) gin_trgm_ops); - CREATE INDEX ON events.pages (response_time); - CREATE INDEX ON events.pages (response_end); + CREATE INDEX pages_response_time_idx ON events.pages (response_time); + CREATE INDEX pages_response_end_idx ON events.pages (response_end); CREATE INDEX pages_path_gin_idx ON events.pages USING GIN (path gin_trgm_ops); CREATE INDEX pages_path_idx ON events.pages (path); CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; @@ -749,10 +749,10 @@ $$ selector text DEFAULT '' NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.clicks (session_id); - CREATE INDEX ON events.clicks (label); + CREATE INDEX clicks_session_id_idx ON events.clicks (session_id); + CREATE INDEX clicks_label_idx ON events.clicks (label); CREATE INDEX clicks_label_gin_idx ON events.clicks USING GIN (label gin_trgm_ops); - CREATE INDEX ON events.clicks (timestamp); + CREATE INDEX clicks_timestamp_idx ON events.clicks (timestamp); CREATE INDEX clicks_label_session_id_timestamp_idx ON events.clicks (label, session_id, timestamp); CREATE INDEX clicks_url_idx ON events.clicks (url); CREATE INDEX clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops); @@ -769,11 +769,11 @@ $$ value text DEFAULT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.inputs (session_id); - CREATE INDEX ON events.inputs (label, value); + CREATE INDEX inputs_session_id_idx ON events.inputs (session_id); + CREATE INDEX inputs_label_idx ON events.inputs (label, value); CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); CREATE INDEX inputs_label_idx ON events.inputs (label); - CREATE INDEX ON events.inputs (timestamp); + CREATE INDEX inputs_timestamp_idx ON events.inputs (timestamp); CREATE INDEX inputs_label_session_id_timestamp_idx ON events.inputs (label, session_id, timestamp); CREATE TABLE events.errors @@ -784,8 +784,8 @@ $$ error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.errors (session_id); - CREATE INDEX ON events.errors (timestamp); + CREATE INDEX errors_session_id_idx ON events.errors (session_id); + CREATE INDEX errors_timestamp_idx ON events.errors (timestamp); CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); @@ -800,9 +800,9 @@ $$ name text NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.graphql (name); + CREATE INDEX graphql_name_idx ON events.graphql (name); CREATE INDEX graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); - CREATE INDEX ON events.graphql (timestamp); + CREATE INDEX graphql_timestamp_idx ON events.graphql (timestamp); CREATE TABLE events.state_actions ( @@ -812,9 +812,9 @@ $$ name text NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.state_actions (name); + CREATE INDEX state_actions_name_idx ON events.state_actions (name); CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); - CREATE INDEX ON events.state_actions (timestamp); + CREATE INDEX state_actions_timestamp_idx ON events.state_actions (timestamp); CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); @@ -837,13 +837,13 @@ $$ decoded_body_size integer NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX ON events.resources (session_id); - CREATE INDEX ON events.resources (status); - CREATE INDEX ON events.resources (type); - CREATE INDEX ON events.resources (duration) WHERE duration > 0; - CREATE INDEX ON events.resources (url_host); - CREATE INDEX ON events.resources (timestamp); - CREATE INDEX ON events.resources (success); + CREATE INDEX resources_session_id_idx ON events.resources (session_id); + CREATE INDEX resources_status_idx ON events.resources (status); + CREATE INDEX resources_type_idx ON events.resources (type); + CREATE INDEX resources_duration_durationgt0_idx ON events.resources (duration) WHERE duration > 0; + CREATE INDEX resources_url_host_idx ON events.resources (url_host); + CREATE INDEX resources_timestamp_idx ON events.resources (timestamp); + CREATE INDEX resources_success_idx ON events.resources (success); CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); CREATE INDEX resources_url_idx ON events.resources (url); @@ -909,8 +909,8 @@ $$ start_at timestamp NOT NULL, errors text NULL ); - CREATE INDEX ON jobs (status); - CREATE INDEX ON jobs (start_at); + CREATE INDEX jobs_status_idx ON jobs (status); + CREATE INDEX jobs_start_at_idx ON jobs (start_at); CREATE INDEX jobs_project_id_idx ON jobs (project_id); From d6cdb28123eabe4957e9d2d3730fb45832ac6bbb Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 3 Jan 2022 10:22:04 +0100 Subject: [PATCH 040/206] feat(DB): indexes for parent_error_id --- ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 1 + ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 1 + scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 1 + scripts/helm/db/init_dbs/postgresql/init_schema.sql | 1 + 4 files changed, 4 insertions(+) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 39fa107da..28d26a459 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -72,4 +72,5 @@ DROP INDEX IF EXISTS sessions_project_id_user_country_idx1; ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; ALTER INDEX IF EXISTS events.resources_duration_idx RENAME TO resources_duration_durationgt0_idx; DROP INDEX IF EXISTS projects_project_key_idx1; +CREATE INDEX IF NOT EXISTS errors_parent_error_id_idx ON errors (parent_error_id); COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 17d6c09c4..16173b0f3 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -497,6 +497,7 @@ $$ CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; CREATE INDEX errors_error_id_idx ON errors (error_id); + CREATE INDEX errors_parent_error_id_idx ON errors (parent_error_id); CREATE TABLE user_favorite_errors ( diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index eb691bfbf..95d8bcb64 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -23,4 +23,5 @@ DROP INDEX IF EXISTS sessions_project_id_user_country_idx1; ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; ALTER INDEX IF EXISTS events.resources_duration_idx RENAME TO resources_duration_durationgt0_idx; DROP INDEX IF EXISTS projects_project_key_idx1; +CREATE INDEX IF NOT EXISTS errors_parent_error_id_idx ON errors (parent_error_id); COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 793e53699..03653be23 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -462,6 +462,7 @@ $$ CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; CREATE INDEX errors_error_id_idx ON errors (error_id); + CREATE INDEX errors_parent_error_id_idx ON errors (parent_error_id); CREATE TABLE user_favorite_errors ( From 549258d0475fd6d29c5a212659a8f2646f436620 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 3 Jan 2022 15:37:49 +0530 Subject: [PATCH 041/206] change(ui) - errors nav and other route changes --- frontend/app/Router.js | 4 +- frontend/app/components/Errors/List/List.js | 13 ++--- frontend/app/components/Session/LivePlayer.js | 20 ++++--- frontend/app/components/Session/Session.js | 4 +- .../components/Session_/PlayerBlockHeader.js | 53 +++++++++++-------- frontend/app/duck/errors.js | 13 ++++- 6 files changed, 68 insertions(+), 39 deletions(-) diff --git a/frontend/app/Router.js b/frontend/app/Router.js index 06b71e3cb..7e5511231 100644 --- a/frontend/app/Router.js +++ b/frontend/app/Router.js @@ -24,6 +24,7 @@ import * as routes from './routes'; import { OB_DEFAULT_TAB } from 'App/routes'; import Signup from './components/Signup/Signup'; import { fetchTenants } from 'Duck/user'; +import { setSessionPath } from 'Duck/sessions'; const BugFinder = withSiteIdUpdater(BugFinderPure); const Dashboard = withSiteIdUpdater(DashboardPure); @@ -73,7 +74,7 @@ const ONBOARDING_REDIRECT_PATH = routes.onboarding(OB_DEFAULT_TAB); onboarding: state.getIn([ 'user', 'onboarding' ]) }; }, { - fetchUserInfo, fetchTenants + fetchUserInfo, fetchTenants, setSessionPath }) class Router extends React.Component { state = { @@ -96,6 +97,7 @@ class Router extends React.Component { } componentDidUpdate(prevProps, prevState) { + this.props.setSessionPath(prevProps.location.pathname) if (prevProps.email !== this.props.email && !this.props.email) { this.props.fetchTenants(); } diff --git a/frontend/app/components/Errors/List/List.js b/frontend/app/components/Errors/List/List.js index 70b0953fb..cb0ffd55a 100644 --- a/frontend/app/components/Errors/List/List.js +++ b/frontend/app/components/Errors/List/List.js @@ -2,7 +2,7 @@ import cn from 'classnames'; import { connect } from 'react-redux'; import { Set, List as ImmutableList } from "immutable"; import { NoContent, Loader, Checkbox, LoadMoreButton, IconButton, Input, DropdownPlain } from 'UI'; -import { merge, resolve,unresolve,ignore } from "Duck/errors"; +import { merge, resolve, unresolve, ignore, updateCurrentPage } from "Duck/errors"; import { applyFilter } from 'Duck/filters'; import { IGNORED, RESOLVED, UNRESOLVED } from 'Types/errorInfo'; import SortDropdown from 'Components/BugFinder/Filters/SortDropdown'; @@ -30,18 +30,19 @@ const sortOptions = Object.entries(sortOptionsMap) state.getIn(["errors", "unresolve", "loading"]), ignoreLoading: state.getIn([ "errors", "ignore", "loading" ]), mergeLoading: state.getIn([ "errors", "merge", "loading" ]), + currentPage: state.getIn(["errors", "currentPage"]), }), { merge, resolve, unresolve, ignore, - applyFilter + applyFilter, + updateCurrentPage, }) export default class List extends React.PureComponent { state = { checkedAll: false, checkedIds: Set(), - showPages: 1, sort: {} } @@ -106,7 +107,7 @@ export default class List extends React.PureComponent { this.applyToAllChecked(this.props.ignore); } - addPage = () => this.setState({ showPages: this.state.showPages + 1 }) + addPage = () => this.props.updateCurrentPage(this.props.currentPage + 1) writeOption = (e, { name, value }) => { const [ sort, order ] = value.split('-'); @@ -123,16 +124,16 @@ export default class List extends React.PureComponent { resolveToggleLoading, mergeLoading, onFilterChange, + currentPage, } = this.props; const { checkedAll, checkedIds, - showPages, sort } = this.state; const someLoading = loading || ignoreLoading || resolveToggleLoading || mergeLoading; const currentCheckedIds = this.currentCheckedIds(); - const displayedCount = Math.min(showPages * PER_PAGE, list.size); + const displayedCount = Math.min(currentPage * PER_PAGE, list.size); let _list = sort.sort ? list.sortBy(i => i[sort.sort]) : list; _list = sort.order === 'desc' ? _list.reverse() : _list; diff --git a/frontend/app/components/Session/LivePlayer.js b/frontend/app/components/Session/LivePlayer.js index 34c56721e..878ddf10d 100644 --- a/frontend/app/components/Session/LivePlayer.js +++ b/frontend/app/components/Session/LivePlayer.js @@ -71,13 +71,17 @@ export default withRequest({ dataName: 'assistCredendials', loadingName: 'loadingCredentials', })(withPermissions(['SESSION_REPLAY', 'ASSIST_LIVE'], '', true)(connect( - state => ({ - session: state.getIn([ 'sessions', 'current' ]), - showAssist: state.getIn([ 'sessions', 'showChatWindow' ]), - jwt: state.get('jwt'), - fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), - hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), - isEnterprise: state.getIn([ 'user', 'client', 'edition' ]) === 'ee', - }), + state => { + const isAssist = state.getIn(['sessions', 'activeTab']).type === 'live'; + const hasSessioPath = state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'); + return { + session: state.getIn([ 'sessions', 'current' ]), + showAssist: state.getIn([ 'sessions', 'showChatWindow' ]), + jwt: state.get('jwt'), + fullscreen: state.getIn([ 'components', 'player', 'fullscreen' ]), + hasSessionsPath: hasSessioPath && !isAssist, + isEnterprise: state.getIn([ 'user', 'client', 'edition' ]) === 'ee', + } + }, { toggleFullscreen, closeBottomBlock }, )(WebPlayer))); diff --git a/frontend/app/components/Session/Session.js b/frontend/app/components/Session/Session.js index e7af00642..0138e7e50 100644 --- a/frontend/app/components/Session/Session.js +++ b/frontend/app/components/Session/Session.js @@ -60,12 +60,14 @@ function Session({ export default withPermissions(['SESSION_REPLAY'], '', true)(connect((state, props) => { const { match: { params: { sessionId } } } = props; + const isAssist = state.getIn(['sessions', 'activeTab']).type === 'live'; + const hasSessiosPath = state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'); return { sessionId, loading: state.getIn([ 'sessions', 'loading' ]), hasErrors: !!state.getIn([ 'sessions', 'errors' ]), session: state.getIn([ 'sessions', 'current' ]), - hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), + hasSessionsPath: hasSessiosPath && !isAssist, }; }, { fetchSession, diff --git a/frontend/app/components/Session_/PlayerBlockHeader.js b/frontend/app/components/Session_/PlayerBlockHeader.js index ea2b63a06..1cade051e 100644 --- a/frontend/app/components/Session_/PlayerBlockHeader.js +++ b/frontend/app/components/Session_/PlayerBlockHeader.js @@ -28,18 +28,23 @@ function capitalise(str) { live: state.live, loading: state.cssLoading || state.messagesLoading, })) -@connect((state, props) => ({ - session: state.getIn([ 'sessions', 'current' ]), - loading: state.getIn([ 'sessions', 'toggleFavoriteRequest', 'loading' ]), - disabled: state.getIn([ 'components', 'targetDefiner', 'inspectorMode' ]) || props.loading, - jiraConfig: state.getIn([ 'issues', 'list' ]).first(), - issuesFetched: state.getIn([ 'issues', 'issuesFetched' ]), - local: state.getIn(['sessions', 'timezone']), - funnelRef: state.getIn(['funnels', 'navRef']), - siteId: state.getIn([ 'user', 'siteId' ]), - funnelPage: state.getIn(['sessions', 'funnelPage']), - hasSessionsPath: state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'), -}), { +@connect((state, props) => { + const isAssist = state.getIn(['sessions', 'activeTab']).type === 'live'; + const hasSessioPath = state.getIn([ 'sessions', 'sessionPath' ]).includes('/sessions'); + return { + session: state.getIn([ 'sessions', 'current' ]), + sessionPath: state.getIn([ 'sessions', 'sessionPath' ]), + loading: state.getIn([ 'sessions', 'toggleFavoriteRequest', 'loading' ]), + disabled: state.getIn([ 'components', 'targetDefiner', 'inspectorMode' ]) || props.loading, + jiraConfig: state.getIn([ 'issues', 'list' ]).first(), + issuesFetched: state.getIn([ 'issues', 'issuesFetched' ]), + local: state.getIn(['sessions', 'timezone']), + funnelRef: state.getIn(['funnels', 'navRef']), + siteId: state.getIn([ 'user', 'siteId' ]), + funnelPage: state.getIn(['sessions', 'funnelPage']), + hasSessionsPath: hasSessioPath && !isAssist, + } +}, { toggleFavorite, fetchListIntegration, setSessionPath }) @withRouter @@ -56,16 +61,22 @@ export default class PlayerBlockHeader extends React.PureComponent { ); backHandler = () => { - const { history, siteId, funnelPage } = this.props; - const funnelId = funnelPage && funnelPage.get('funnelId'); - const issueId = funnelPage && funnelPage.get('issueId'); - if (funnelId || issueId) { - if (issueId) { - history.push(withSiteId(funnelIssueRoute(funnelId, issueId), siteId)) - } else - history.push(withSiteId(funnelRoute(funnelId), siteId)); - } else + const { history, siteId, funnelPage, sessionPath } = this.props; + // alert(sessionPath) + if (sessionPath === history.location.pathname) { history.push(withSiteId(SESSIONS_ROUTE), siteId); + } else { + history.push(sessionPath ? sessionPath : withSiteId(SESSIONS_ROUTE, siteId)); + } + // const funnelId = funnelPage && funnelPage.get('funnelId'); + // const issueId = funnelPage && funnelPage.get('issueId'); + // if (funnelId || issueId) { + // if (issueId) { + // history.push(withSiteId(funnelIssueRoute(funnelId, issueId), siteId)) + // } else + // history.push(withSiteId(funnelRoute(funnelId), siteId)); + // } else + // history.push(withSiteId(SESSIONS_ROUTE), siteId); } toggleFavorite = () => { diff --git a/frontend/app/duck/errors.js b/frontend/app/duck/errors.js index 3f0793103..9e7b552f2 100644 --- a/frontend/app/duck/errors.js +++ b/frontend/app/duck/errors.js @@ -17,6 +17,7 @@ const IGNORE = "errors/IGNORE"; const MERGE = "errors/MERGE"; const TOGGLE_FAVORITE = "errors/TOGGLE_FAVORITE"; const FETCH_TRACE = "errors/FETCH_TRACE"; +const UPDATE_CURRENT_PAGE = "errors/UPDATE_CURRENT_PAGE"; function chartWrapper(chart = []) { return chart.map(point => ({ ...point, count: Math.max(point.count, 0) })); @@ -33,7 +34,8 @@ const initialState = Map({ instance: ErrorInfo(), instanceTrace: List(), stats: Map(), - sourcemapUploaded: true + sourcemapUploaded: true, + currentPage: 1, }); @@ -67,7 +69,8 @@ function reducer(state = initialState, action = {}) { return state.update("list", list => list.filter(e => !ids.includes(e.errorId))); case success(FETCH_NEW_ERRORS_COUNT): return state.set('stats', action.data); - + case UPDATE_CURRENT_PAGE: + return state.set('currentPage', action.page); } return state; } @@ -166,3 +169,9 @@ export function fetchNewErrorsCount(params = {}) { } } +export function updateCurrentPage(page) { + return { + type: 'errors/UPDATE_CURRENT_PAGE', + page, + }; +} From ce3c595a1ca3d59d38868ff507b81f748d49235e Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 3 Jan 2022 15:46:07 +0530 Subject: [PATCH 042/206] fix(ui) - metrics text change --- frontend/app/components/Dashboard/Dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/components/Dashboard/Dashboard.js b/frontend/app/components/Dashboard/Dashboard.js index 2e5cba630..ff91e10a6 100644 --- a/frontend/app/components/Dashboard/Dashboard.js +++ b/frontend/app/components/Dashboard/Dashboard.js @@ -175,7 +175,7 @@ export default class Dashboard extends React.PureComponent {
From a693b9443437173b4f925bc9853196c9c301e8c2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 3 Jan 2022 18:25:38 +0100 Subject: [PATCH 043/206] feat(DB): performance indexes --- ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 7 +++++++ ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 5 +++++ scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 7 +++++++ scripts/helm/db/init_dbs/postgresql/init_schema.sql | 5 +++++ 4 files changed, 24 insertions(+) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 28d26a459..808bdb9cc 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -73,4 +73,11 @@ ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; ALTER INDEX IF EXISTS events.resources_duration_idx RENAME TO resources_duration_durationgt0_idx; DROP INDEX IF EXISTS projects_project_key_idx1; CREATE INDEX IF NOT EXISTS errors_parent_error_id_idx ON errors (parent_error_id); + +CREATE INDEX IF NOT EXISTS performance_session_id_idx ON events.performance (session_id); +CREATE INDEX IF NOT EXISTS performance_timestamp_idx ON events.performance (timestamp); +CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); +CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; +CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; + COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 16173b0f3..24d0b6423 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -916,6 +916,11 @@ $$ max_used_js_heap_size bigint NOT NULL, PRIMARY KEY (session_id, message_id) ); + CREATE INDEX performance_session_id_idx ON events.performance (session_id); + CREATE INDEX performance_timestamp_idx ON events.performance (timestamp); + CREATE INDEX performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); + CREATE INDEX performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; + CREATE INDEX performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; -- --- autocomplete.sql --- diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 95d8bcb64..a44de93bf 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -24,4 +24,11 @@ ALTER INDEX IF EXISTS platform_idx RENAME TO sessions_platform_idx; ALTER INDEX IF EXISTS events.resources_duration_idx RENAME TO resources_duration_durationgt0_idx; DROP INDEX IF EXISTS projects_project_key_idx1; CREATE INDEX IF NOT EXISTS errors_parent_error_id_idx ON errors (parent_error_id); + +CREATE INDEX IF NOT EXISTS performance_session_id_idx ON events.performance (session_id); +CREATE INDEX IF NOT EXISTS performance_timestamp_idx ON events.performance (timestamp); +CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); +CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; +CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; + COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 03653be23..d40fdab5d 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -878,6 +878,11 @@ $$ max_used_js_heap_size bigint NOT NULL, PRIMARY KEY (session_id, message_id) ); + CREATE INDEX performance_session_id_idx ON events.performance (session_id); + CREATE INDEX performance_timestamp_idx ON events.performance (timestamp); + CREATE INDEX performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); + CREATE INDEX performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; + CREATE INDEX performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; -- --- autocomplete.sql --- From e94de67738f13e3fa6fd50ac074b057cd2fe5814 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 3 Jan 2022 18:27:29 +0100 Subject: [PATCH 044/206] feat(api): performance largest_contentful_paint --- api/chalicelib/core/performance_event.py | 3 ++- api/schemas.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/performance_event.py b/api/chalicelib/core/performance_event.py index 1311614d9..76633ce40 100644 --- a/api/chalicelib/core/performance_event.py +++ b/api/chalicelib/core/performance_event.py @@ -10,5 +10,6 @@ def get_col(perf: schemas.PerformanceEventType): "extraJoin": "events.performance"}, schemas.PerformanceEventType.fetch_failed: {"column": "success", "extraJoin": None}, # schemas.PerformanceEventType.fetch_duration: {"column": "duration", "extraJoin": None}, - # schemas.PerformanceEventType.location_largest_contentful_paint_time: "timestamp" + schemas.PerformanceEventType.location_largest_contentful_paint_time: {"column": "first_contentful_paint_time", + "extraJoin": None} }.get(perf) diff --git a/api/schemas.py b/api/schemas.py index 76b4f0ce8..fe5296e52 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -442,7 +442,7 @@ class IssueType(str, Enum): class _SessionSearchEventRaw(BaseModel): - custom: Optional[List[Union[int, str]]] = Field(None) + custom: Optional[List[Union[int, str]]] = Field(None, min_items=1) customOperator: Optional[MathOperator] = Field(None) key: Optional[str] = Field(None) value: Union[str, List[str]] = Field(...) @@ -455,9 +455,9 @@ class _SessionSearchEventRaw(BaseModel): if isinstance(values.get("type"), PerformanceEventType): if values.get("type") == PerformanceEventType.fetch_failed: return values - assert values.get("custom") is not None, "custom should not be None for PerformanceEventType" + assert values.get("custom") is not None, "custom should not be null for PerformanceEventType" assert values.get("customOperator") is not None \ - , "customOperator should not be None for PerformanceEventType" + , "customOperator should not be null for PerformanceEventType" if values["type"] == PerformanceEventType.time_between_events: assert len(values.get("value", [])) == 2, \ f"must provide 2 Events as value for {PerformanceEventType.time_between_events}" From 8d2f688986c2409dc400e9f44f73eb4580061c7d Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Tue, 4 Jan 2022 16:03:09 +0530 Subject: [PATCH 045/206] fix(ui) - compile errors --- .../app/player/MessageDistributor/managers/AssistManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index fc762dcc7..03b2b9512 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -147,13 +147,13 @@ export default class AssistManager { return; } this.setStatus(ConnectionStatus.Connecting) + // @ts-ignore const urlObject = new URL(window.ENV.API_EDP) import('peerjs').then(({ default: Peer }) => { const _config = { - // @ts-ignore host: urlObject.hostname, path: '/assist', - port: urlObject.port === "" ? location.protocol === 'https:' ? 443 : 80 : urlObject.port, + port: urlObject.port === "" ? (location.protocol === 'https:' ? 443 : 80 ): parseInt(urlObject.port), } if (this.config) { From 7aafd8bc83d00a7c0b08a5ba8dd2b0e63a832b5a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 5 Jan 2022 19:21:31 +0100 Subject: [PATCH 046/206] feat(api): custom metrics CRUD feat(api): EE fixed add/edit project feat(api): refactored sessions search --- api/chalicelib/core/custom_metrics.py | 200 +++++ api/chalicelib/core/sessions.py | 1034 ++++++++++++------------- api/routers/core.py | 38 +- api/schemas.py | 30 + ee/api/chalicelib/core/projects.py | 13 +- 5 files changed, 788 insertions(+), 527 deletions(-) create mode 100644 api/chalicelib/core/custom_metrics.py diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py new file mode 100644 index 000000000..f716c4799 --- /dev/null +++ b/api/chalicelib/core/custom_metrics.py @@ -0,0 +1,200 @@ +import json + +import schemas +from chalicelib.utils import helper, pg_client +from chalicelib.utils.TimeUTC import TimeUTC + + +def try_live(project_id,data: schemas.CreateCustomMetricsSchema): + with pg_client.PostgresClient() as cur: + _data = {} + for i, s in enumerate(data.series): + for k in s.dict().keys(): + _data[f"{k}_{i}"] = s.__getattribute__(k) + _data[f"index_{i}"] = i + _data[f"filter_{i}"] = s.filter.json() + series_len = len(data.series) + data.series = None + params = { "project_id": project_id, **data.dict(), **_data} + query = cur.mogrify(f"""\ + WITH m AS (INSERT INTO metrics (project_id, user_id, title) + VALUES (%(project_id)s, %(user_id)s, %(title)s) + RETURNING *) + INSERT + INTO metric_series(metric_id, index, title, filter) + VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(title_{i})s, %(filter_{i})s::jsonb)" + for i in range(series_len)])} + RETURNING metric_id;""", params) + + cur.execute( + query + ) + r = cur.fetchone() + r = helper.dict_to_camel_case(r) + return {"data": r} + +def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): + with pg_client.PostgresClient() as cur: + _data = {} + for i, s in enumerate(data.series): + for k in s.dict().keys(): + _data[f"{k}_{i}"] = s.__getattribute__(k) + _data[f"index_{i}"] = i + _data[f"filter_{i}"] = s.filter.json() + series_len = len(data.series) + data.series = None + params = {"user_id": user_id, "project_id": project_id, **data.dict(), **_data} + query = cur.mogrify(f"""\ + WITH m AS (INSERT INTO metrics (project_id, user_id, title) + VALUES (%(project_id)s, %(user_id)s, %(title)s) + RETURNING *) + INSERT + INTO metric_series(metric_id, index, title, filter) + VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(title_{i})s, %(filter_{i})s::jsonb)" + for i in range(series_len)])} + RETURNING metric_id;""", params) + + cur.execute( + query + ) + r = cur.fetchone() + r = helper.dict_to_camel_case(r) + return {"data": r} + + +def __get_series_id(metric_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT series_id + FROM metric_series + WHERE metric_series.metric_id = %(metric_id)s + AND metric_series.deleted_at ISNULL;""", + {"metric_id": metric_id} + ) + ) + rows = cur.fetchall() + return [r["series_id"] for r in rows] + + +def update(metric_id, user_id, data: schemas.UpdateCustomMetricsSchema): + series_ids = __get_series_id(metric_id) + n_series = [] + d_series_ids = [] + u_series = [] + u_series_ids = [] + params = {"metric_id": metric_id, "is_public": data.is_public, "title": data.title} + for i, s in enumerate(data.series): + prefix = "u_" + if s.series_id is None: + n_series.append({"i": i, "s": s}) + prefix = "n_" + else: + u_series.append({"i": i, "s": s}) + u_series_ids.append(s.series_id) + ns = s.dict() + for k in ns.keys(): + if k == "filter": + ns[k] = json.dumps(ns[k]) + params[f"{prefix}{k}_{i}"] = ns[k] + for i in series_ids: + if i not in u_series_ids: + d_series_ids.append(i) + params["d_series_ids"] = tuple(d_series_ids) + + with pg_client.PostgresClient() as cur: + sub_queries = [] + if len(n_series) > 0: + sub_queries.append(f"""\ + n AS (INSERT INTO metric_series (metric_id, index, title, filter) + VALUES {",".join([f"(%(metric_id)s, %(n_index_{s['i']})s, %(n_title_{s['i']})s, %(n_filter_{s['i']})s::jsonb)" + for s in n_series])} + RETURNING 1)""") + if len(u_series) > 0: + sub_queries.append(f"""\ + u AS (UPDATE metric_series + SET title=series.title, + filter=series.filter, + index=series.filter.index + FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_title_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" + for s in n_series])}) AS series(series_id, index, title, filter) + WHERE metric_id =%(metric_id)s AND series_id=series.series_id + RETURNING 1)""") + if len(d_series_ids) > 0: + sub_queries.append("""\ + d AS (DELETE FROM metric_series WHERE metric_id =%(metric_id)s AND series_id IN %(d_series_ids)s + RETURNING 1)""") + query = cur.mogrify(f"""\ + {"WITH " if len(sub_queries) > 0 else ""}{",".join(sub_queries)} + UPDATE metrics + SET title = %(title)s, is_public= %(is_public)s WHERE metric_id = %(metric_id)s + RETURNING metric_id;""", params) + cur.execute( + query + ) + r = cur.fetchone() + r = helper.dict_to_camel_case(r) + return r + + +def get_all(project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT * + FROM metrics + LEFT JOIN LATERAL (SELECT jsonb_agg(metric_series.* ORDER BY index) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE metrics.project_id = %(project_id)s + AND metrics.deleted_at ISNULL + AND (user_id = %(user_id)s OR is_public) + ORDER BY created_at;""", + {"project_id": project_id, "user_id": user_id} + ) + ) + rows = cur.fetchall() + for r in rows: + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + rows = helper.list_to_camel_case(rows) + return rows + + +def delete(project_id, metric_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.metrics + SET deleted_at = timezone('utc'::text, now()) + WHERE project_id = %(project_id)s + AND metric_id = %(metric_id)s;""", + {"metric_id": metric_id, "project_id": project_id, "user_id": user_id}) + ) + + return {"state": "success"} + + +def get(metric_id, project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT * + FROM metrics + LEFT JOIN LATERAL (SELECT jsonb_agg(metric_series.* ORDER BY index) AS series + FROM metric_series + WHERE metric_series.metric_id = metrics.metric_id + AND metric_series.deleted_at ISNULL + ) AS metric_series ON (TRUE) + WHERE metrics.project_id = %(project_id)s + AND metrics.deleted_at ISNULL + AND (metrics.user_id = %(user_id)s OR metrics.is_public) + AND metrics.metric_id = %(metric_id)s + ORDER BY created_at;""", + {"metric_id": metric_id, "project_id": project_id, "user_id": user_id} + ) + ) + row = cur.fetchone() + row["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8517b1161..9f4213add 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -162,519 +162,11 @@ def _isAny_opreator(op: schemas.SearchEventOperator): @dev.timed def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, favorite_only=False, errors_only=False, - error_status="ALL", - count_only=False, issue=None): + error_status="ALL", count_only=False, issue=None): + full_args, query_part, sort = search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, + user_id) + with pg_client.PostgresClient() as cur: - ss_constraints = [] - full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, - "projectId": project_id, "userId": user_id} - extra_constraints = [ - "s.project_id = %(project_id)s", - "s.duration IS NOT NULL" - ] - extra_from = "" - fav_only_join = "" - if favorite_only and not errors_only: - fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" - extra_constraints.append("fs.user_id = %(userId)s") - full_args["userId"] = user_id - events_query_part = "" - - if len(data.filters) > 0: - meta_keys = None - for i, f in enumerate(data.filters): - if not isinstance(f.value, list): - f.value = [f.value] - if len(f.value) == 0 or f.value[0] is None: - continue - filter_type = f.type - # f.value = __get_sql_value_multiple(f.value) - f.value = helper.values_for_operator(value=f.value, op=f.operator) - f_k = f"f_value{i}" - full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} - op = __get_sql_operator(f.operator) \ - if filter_type not in [schemas.FilterType.events_count] else f.operator - is_any = _isAny_opreator(f.operator) - is_not = False - if __is_negation_operator(f.operator): - is_not = True - # op = __reverse_sql_operator(op) - if filter_type == schemas.FilterType.user_browser: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - - elif filter_type in [schemas.FilterType.utm_source]: - if is_any: - extra_constraints.append('s.utm_source IS NOT NULL') - ss_constraints.append('ms.utm_source IS NOT NULL') - else: - extra_constraints.append( - _multiple_conditions(f's.utm_source {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.utm_source {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.utm_medium]: - if is_any: - extra_constraints.append('s.utm_medium IS NOT NULL') - ss_constraints.append('ms.utm_medium IS NOT NULL') - else: - extra_constraints.append( - _multiple_conditions(f's.utm_medium {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.utm_medium {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.utm_campaign]: - if is_any: - extra_constraints.append('s.utm_campaign IS NOT NULL') - ss_constraints.append('ms.utm_campaign IS NOT NULL') - else: - extra_constraints.append( - _multiple_conditions(f's.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, - value_key=f_k)) - - elif filter_type == schemas.FilterType.duration: - if len(f.value) > 0 and f.value[0] is not None: - extra_constraints.append("s.duration >= %(minDuration)s") - ss_constraints.append("ms.duration >= %(minDuration)s") - full_args["minDuration"] = f.value[0] - if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: - extra_constraints.append("s.duration <= %(maxDuration)s") - ss_constraints.append("ms.duration <= %(maxDuration)s") - full_args["maxDuration"] = f.value[1] - elif filter_type == schemas.FilterType.referrer: - # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" - extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" - # op = __get_sql_operator_multiple(f.operator) - extra_constraints.append( - _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type == events.event_type.METADATA.ui_type: - # get metadata list only if you need it - if meta_keys is None: - meta_keys = metadata.get(project_id=project_id) - meta_keys = {m["key"]: m["index"] for m in meta_keys} - # op = __get_sql_operator(f.operator) - if f.key in meta_keys.keys(): - extra_constraints.append( - _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", - f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", - f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type in [schemas.FilterType.user_anonymous_id, - schemas.FilterType.user_anonymous_id_ios]: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) - elif filter_type == schemas.FilterType.platform: - # op = __get_sql_operator(f.operator) - extra_constraints.append( - _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == schemas.FilterType.issue: - extra_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, - value_key=f_k)) - elif filter_type == schemas.FilterType.events_count: - extra_constraints.append( - _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - ss_constraints.append( - _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, - value_key=f_k)) - - # --------------------------------------------------------------------------- - if len(data.events) > 0: - # ss_constraints = [s.decode('UTF-8') for s in ss_constraints] - events_query_from = [] - event_index = 0 - or_events = data.events_order == schemas.SearchEventOrder._or - # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " - events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " - for i, event in enumerate(data.events): - event_type = event.type - is_any = _isAny_opreator(event.operator) - if not isinstance(event.value, list): - event.value = [event.value] - op = __get_sql_operator(event.operator) - is_not = False - if __is_negation_operator(event.operator): - is_not = True - op = __reverse_sql_operator(op) - if event_index == 0 or or_events: - event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" - event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", - "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", - "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] - else: - event_from = "%s" - event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", - "main.session_id=event_0.session_id"] - if data.events_order == schemas.SearchEventOrder._then: - event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") - e_k = f"e_value{i}" - if event.type != schemas.PerformanceEventType.time_between_events: - event.value = helper.values_for_operator(value=event.value, op=event.operator) - full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} - - # if event_type not in list(events.SUPPORTED_TYPES.keys()) \ - # or event.value in [None, "", "*"] \ - # and (event_type != events.event_type.ERROR.ui_type \ - # or event_type != events.event_type.ERROR_IOS.ui_type): - # continue - if event_type == events.event_type.CLICK.ui_type: - event_from = event_from % f"{events.event_type.CLICK.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, - value_key=e_k)) - - elif event_type == events.event_type.INPUT.ui_type: - event_from = event_from % f"{events.event_type.INPUT.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, - value_key=e_k)) - if event.custom is not None and len(event.custom) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, - value_key=f"custom{i}")) - full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} - - elif event_type == events.event_type.LOCATION.ui_type: - event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.CUSTOM.ui_type: - event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.event_type.REQUEST.ui_type: - event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.event_type.GRAPHQL.ui_type: - event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, - value_key=e_k)) - elif event_type == events.event_type.STATEACTION.ui_type: - event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.ERROR.ui_type: - # if event.source in [None, "*", ""]: - # event.source = "js_exception" - event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" - if event.value not in [None, "*", ""]: - if not is_any: - event_where.append(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)") - if event.source not in [None, "*", ""]: - event_where.append(f"main1.source = %(source)s") - full_args["source"] = event.source - elif event.source not in [None, "*", ""]: - event_where.append(f"main1.source = %(source)s") - full_args["source"] = event.source - - # ----- IOS - elif event_type == events.event_type.CLICK_IOS.ui_type: - event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - - elif event_type == events.event_type.INPUT_IOS.ui_type: - event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - if event.custom is not None and len(event.custom) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, - value_key="custom{i}")) - full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} - elif event_type == events.event_type.VIEW_IOS.ui_type: - event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.CUSTOM_IOS.ui_type: - event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.REQUEST_IOS.ui_type: - event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", - event.value, value_key=e_k)) - elif event_type == events.event_type.ERROR_IOS.ui_type: - event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" - if not is_any and event.value not in [None, "*", ""]: - event_where.append( - _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", - event.value, value_key=e_k)) - elif event_type == schemas.PerformanceEventType.fetch_failed: - event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - event.value, value_key=e_k)) - col = performance_event.get_col(event_type) - colname = col["column"] - event_where.append(f"main.{colname} = FALSE") - # elif event_type == schemas.PerformanceEventType.fetch_duration: - # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " - # if not is_any: - # event_where.append( - # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", - # event.value, value_key=e_k)) - # col = performance_event.get_col(event_type) - # colname = col["column"] - # tname = "main" - # e_k += "_custom" - # full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - # _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - # event.custom, value_key=e_k)) - elif event_type in [schemas.PerformanceEventType.location_dom_complete, - schemas.PerformanceEventType.location_largest_contentful_paint_time, - schemas.PerformanceEventType.location_ttfb, - schemas.PerformanceEventType.location_avg_cpu_load, - schemas.PerformanceEventType.location_avg_memory_usage - ]: - event_from = event_from % f"{events.event_type.LOCATION.table} AS main " - col = performance_event.get_col(event_type) - colname = col["column"] - tname = "main" - if col.get("extraJoin") is not None: - tname = "ej" - event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" - event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", - f"{tname}.timestamp <= %(endDate)s"] - if not is_any: - event_where.append( - _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", - event.value, value_key=e_k)) - e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - - event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) - elif event_type == schemas.PerformanceEventType.time_between_events: - event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " - if not isinstance(event.value[0].value, list): - event.value[0].value = [event.value[0].value] - if not isinstance(event.value[1].value, list): - event.value[1].value = [event.value[1].value] - event.value[0].value = helper.values_for_operator(value=event.value[0].value, - op=event.value[0].operator) - event.value[1].value = helper.values_for_operator(value=event.value[1].value, - op=event.value[0].operator) - e_k1 = e_k + "_e1" - e_k2 = e_k + "_e2" - full_args = {**full_args, - **_multiple_values(event.value[0].value, value_key=e_k1), - **_multiple_values(event.value[1].value, value_key=e_k2)} - s_op = __get_sql_operator(event.value[0].operator) - event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] - if event_index > 0 and not or_events: - event_where.append("main2.session_id=event_0.session_id") - event_where.append( - _multiple_conditions( - f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", - event.value[0].value, value_key=e_k1)) - s_op = __get_sql_operator(event.value[1].operator) - event_where.append( - _multiple_conditions( - f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", - event.value[1].value, value_key=e_k2)) - - e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} - event_where.append( - _multiple_conditions(f"main2.timestamp - main.timestamp {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) - - - else: - continue - if event_index == 0 or or_events: - event_where += ss_constraints - if is_not: - if event_index == 0 or or_events: - events_query_from.append(f"""\ - (SELECT - session_id, - 0 AS timestamp - FROM sessions - WHERE EXISTS(SELECT session_id - FROM {event_from} - WHERE {" AND ".join(event_where)} - AND sessions.session_id=ms.session_id) IS FALSE - AND project_id = %(projectId)s - AND start_ts >= %(startDate)s - AND start_ts <= %(endDate)s - AND duration IS NOT NULL - ) {"" if or_events else ("AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ - """) - else: - events_query_from.append(f"""\ - (SELECT - event_0.session_id, - event_{event_index - 1}.timestamp AS timestamp - WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE - ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ - """) - else: - events_query_from.append(f"""\ - (SELECT main.session_id, MIN(main.timestamp) AS timestamp - FROM {event_from} - WHERE {" AND ".join(event_where)} - GROUP BY 1 - ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ - """) - event_index += 1 - if event_index > 0: - if or_events: - events_query_part = f"""SELECT - session_id, - MIN(timestamp) AS first_event_ts, - MAX(timestamp) AS last_event_ts - FROM ({events_joiner.join(events_query_from)}) AS u - GROUP BY 1 - {fav_only_join}""" - else: - events_query_part = f"""SELECT - event_0.session_id, - MIN(event_0.timestamp) AS first_event_ts, - MAX(event_{event_index - 1}.timestamp) AS last_event_ts - FROM {events_joiner.join(events_query_from)} - GROUP BY 1 - {fav_only_join}""" - else: - data.events = [] - - # --------------------------------------------------------------------------- - - if data.startDate is not None: - extra_constraints.append("s.start_ts >= %(startDate)s") - - if data.endDate is not None: - extra_constraints.append("s.start_ts <= %(endDate)s") - - # if data.platform is not None: - # if data.platform == schemas.PlatformType.mobile: - # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") - # elif data.platform == schemas.PlatformType.desktop: - # extra_constraints.append( - # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") - - order = "DESC" - if data.order is not None: - order = data.order - sort = 'session_id' - if data.sort is not None and data.sort != "session_id": - sort += " " + order + "," + helper.key_to_snake_case(data.sort) - else: - sort = 'session_id' - - if errors_only: - extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" - extra_constraints.append(b"ser.source = 'js_exception'") - if error_status != "ALL": - extra_constraints.append(cur.mogrify("ser.status = %(status)s", {"status": error_status.lower()})) - if favorite_only: - extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" - extra_constraints.append(cur.mogrify("ufe.user_id = %(user_id)s", {"user_id": user_id})) - - # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] - if not favorite_only and not errors_only: - extra_from += """LEFT JOIN (SELECT user_id, session_id - FROM public.user_favorite_sessions - WHERE user_id = %(userId)s) AS favorite_sessions - USING (session_id)""" - extra_join = "" - if issue is not None: - extra_join = cur.mogrify(""" - INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) - WHERE issues.session_id=f.session_id - AND p_issues.type=%(type)s - AND p_issues.context_string=%(contextString)s - AND timestamp >= f.first_event_ts - AND timestamp <= f.last_event_ts) AS issues ON(TRUE) - """, {"contextString": issue["contextString"], "type": issue["type"]}).decode('UTF-8') - - query_part = f"""\ - FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} - {extra_join} - {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} - {extra_from} - WHERE - - {" AND ".join(extra_constraints)}""" - if errors_only: main_query = cur.mogrify(f"""SELECT DISTINCT er.error_id, ser.status, ser.parent_error_id, ser.payload, COALESCE((SELECT TRUE @@ -685,21 +177,19 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f FROM public.user_viewed_errors AS ve WHERE er.error_id = ve.error_id AND ve.user_id = %(userId)s LIMIT 1), FALSE) AS viewed - {query_part};""", - full_args) + {query_part};""", full_args) elif count_only: - main_query = cur.mogrify( - f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, COUNT(DISTINCT s.user_uuid) AS count_users - {query_part};""", - full_args) + main_query = cur.mogrify(f"""SELECT COUNT(DISTINCT s.session_id) AS count_sessions, + COUNT(DISTINCT s.user_uuid) AS count_users + {query_part};""", full_args) else: main_query = cur.mogrify(f"""SELECT COUNT(full_sessions) AS count, COALESCE(JSONB_AGG(full_sessions) FILTER (WHERE rn <= 200), '[]'::JSONB) AS sessions FROM (SELECT *, ROW_NUMBER() OVER (ORDER BY favorite DESC, issue_score DESC, session_id desc, start_ts desc) AS rn FROM (SELECT DISTINCT ON(s.session_id) {SESSION_PROJECTION_COLS} {query_part} ORDER BY s.session_id desc) AS filtred_sessions - ORDER BY favorite DESC, issue_score DESC, {sort} {order}) AS full_sessions;""", + ORDER BY favorite DESC, issue_score DESC, {sort} {data.order}) AS full_sessions;""", full_args) # main_query = cur.mogrify(f"""SELECT * FROM @@ -713,7 +203,7 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f print(main_query) cur.execute(main_query) - print("Executed--------------------") + print("--------------------") if count_only: return helper.dict_to_camel_case(cur.fetchone()) sessions = cur.fetchone() @@ -740,6 +230,510 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f } +def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id): + ss_constraints = [] + full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, + "projectId": project_id, "userId": user_id} + extra_constraints = [ + "s.project_id = %(project_id)s", + "s.duration IS NOT NULL" + ] + extra_from = "" + fav_only_join = "" + if favorite_only and not errors_only: + fav_only_join = "LEFT JOIN public.user_favorite_sessions AS fs ON fs.session_id = s.session_id" + extra_constraints.append("fs.user_id = %(userId)s") + full_args["userId"] = user_id + events_query_part = "" + if len(data.filters) > 0: + meta_keys = None + for i, f in enumerate(data.filters): + if not isinstance(f.value, list): + f.value = [f.value] + if len(f.value) == 0 or f.value[0] is None: + continue + filter_type = f.type + # f.value = __get_sql_value_multiple(f.value) + f.value = helper.values_for_operator(value=f.value, op=f.operator) + f_k = f"f_value{i}" + full_args = {**full_args, **_multiple_values(f.value, value_key=f_k)} + op = __get_sql_operator(f.operator) \ + if filter_type not in [schemas.FilterType.events_count] else f.operator + is_any = _isAny_opreator(f.operator) + is_not = False + if __is_negation_operator(f.operator): + is_not = True + # op = __reverse_sql_operator(op) + if filter_type == schemas.FilterType.user_browser: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_browser {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_os, schemas.FilterType.user_os_ios]: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_os {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_device, schemas.FilterType.user_device_ios]: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_device {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.user_country, schemas.FilterType.user_country_ios]: + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f's.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.user_country {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + + elif filter_type in [schemas.FilterType.utm_source]: + if is_any: + extra_constraints.append('s.utm_source IS NOT NULL') + ss_constraints.append('ms.utm_source IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_source {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_source {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_medium]: + if is_any: + extra_constraints.append('s.utm_medium IS NOT NULL') + ss_constraints.append('ms.utm_medium IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_medium {op} %({f_k})s', f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_medium {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.utm_campaign]: + if is_any: + extra_constraints.append('s.utm_campaign IS NOT NULL') + ss_constraints.append('ms.utm_campaign IS NOT NULL') + else: + extra_constraints.append( + _multiple_conditions(f's.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f'ms.utm_campaign {op} %({f_k})s', f.value, is_not=is_not, + value_key=f_k)) + + elif filter_type == schemas.FilterType.duration: + if len(f.value) > 0 and f.value[0] is not None: + extra_constraints.append("s.duration >= %(minDuration)s") + ss_constraints.append("ms.duration >= %(minDuration)s") + full_args["minDuration"] = f.value[0] + if len(f.value) > 1 and f.value[1] is not None and int(f.value[1]) > 0: + extra_constraints.append("s.duration <= %(maxDuration)s") + ss_constraints.append("ms.duration <= %(maxDuration)s") + full_args["maxDuration"] = f.value[1] + elif filter_type == schemas.FilterType.referrer: + # events_query_part = events_query_part + f"INNER JOIN events.pages AS p USING(session_id)" + extra_from += f"INNER JOIN {events.event_type.LOCATION.table} AS p USING(session_id)" + # op = __get_sql_operator_multiple(f.operator) + extra_constraints.append( + _multiple_conditions(f"p.base_referrer {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == events.event_type.METADATA.ui_type: + # get metadata list only if you need it + if meta_keys is None: + meta_keys = metadata.get(project_id=project_id) + meta_keys = {m["key"]: m["index"] for m in meta_keys} + # op = __get_sql_operator(f.operator) + if f.key in meta_keys.keys(): + extra_constraints.append( + _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type in [schemas.FilterType.user_anonymous_id, + schemas.FilterType.user_anonymous_id_ios]: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_anonymous_id {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type in [schemas.FilterType.rev_id, schemas.FilterType.rev_id_ios]: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.rev_id {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) + elif filter_type == schemas.FilterType.platform: + # op = __get_sql_operator(f.operator) + extra_constraints.append( + _multiple_conditions(f"s.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.user_device_type {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.issue: + extra_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (s.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"%({f_k})s {op} ANY (ms.issue_types)", f.value, is_not=is_not, + value_key=f_k)) + elif filter_type == schemas.FilterType.events_count: + extra_constraints.append( + _multiple_conditions(f"s.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + ss_constraints.append( + _multiple_conditions(f"ms.events_count {op} %({f_k})s", f.value, is_not=is_not, + value_key=f_k)) + # --------------------------------------------------------------------------- + if len(data.events) > 0: + # ss_constraints = [s.decode('UTF-8') for s in ss_constraints] + events_query_from = [] + event_index = 0 + or_events = data.events_order == schemas.SearchEventOrder._or + # events_joiner = " FULL JOIN " if or_events else " INNER JOIN LATERAL " + events_joiner = " UNION " if or_events else " INNER JOIN LATERAL " + for i, event in enumerate(data.events): + event_type = event.type + is_any = _isAny_opreator(event.operator) + if not isinstance(event.value, list): + event.value = [event.value] + op = __get_sql_operator(event.operator) + is_not = False + if __is_negation_operator(event.operator): + is_not = True + op = __reverse_sql_operator(op) + if event_index == 0 or or_events: + event_from = "%s INNER JOIN public.sessions AS ms USING (session_id)" + event_where = ["ms.project_id = %(projectId)s", "main.timestamp >= %(startDate)s", + "main.timestamp <= %(endDate)s", "ms.start_ts >= %(startDate)s", + "ms.start_ts <= %(endDate)s", "ms.duration IS NOT NULL"] + else: + event_from = "%s" + event_where = ["main.timestamp >= %(startDate)s", "main.timestamp <= %(endDate)s", + "main.session_id=event_0.session_id"] + if data.events_order == schemas.SearchEventOrder._then: + event_where.append(f"event_{event_index - 1}.timestamp <= main.timestamp") + e_k = f"e_value{i}" + if event.type != schemas.PerformanceEventType.time_between_events: + event.value = helper.values_for_operator(value=event.value, op=event.operator) + full_args = {**full_args, **_multiple_values(event.value, value_key=e_k)} + + # if event_type not in list(events.SUPPORTED_TYPES.keys()) \ + # or event.value in [None, "", "*"] \ + # and (event_type != events.event_type.ERROR.ui_type \ + # or event_type != events.event_type.ERROR_IOS.ui_type): + # continue + if event_type == events.event_type.CLICK.ui_type: + event_from = event_from % f"{events.event_type.CLICK.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK.column} {op} %({e_k})s", event.value, + value_key=e_k)) + + elif event_type == events.event_type.INPUT.ui_type: + event_from = event_from % f"{events.event_type.INPUT.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, + value_key=e_k)) + if event.custom is not None and len(event.custom) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + value_key=f"custom{i}")) + full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} + + elif event_type == events.event_type.LOCATION.ui_type: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.REQUEST.ui_type: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.GRAPHQL.ui_type: + event_from = event_from % f"{events.event_type.GRAPHQL.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.GRAPHQL.column} {op} %({e_k})s", event.value, + value_key=e_k)) + elif event_type == events.event_type.STATEACTION.ui_type: + event_from = event_from % f"{events.event_type.STATEACTION.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.STATEACTION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR.ui_type: + # if event.source in [None, "*", ""]: + # event.source = "js_exception" + event_from = event_from % f"{events.event_type.ERROR.table} AS main INNER JOIN public.errors AS main1 USING(error_id)" + if event.value not in [None, "*", ""]: + if not is_any: + event_where.append(f"(main1.message {op} %({e_k})s OR main1.name {op} %({e_k})s)") + if event.source not in [None, "*", ""]: + event_where.append(f"main1.source = %(source)s") + full_args["source"] = event.source + elif event.source not in [None, "*", ""]: + event_where.append(f"main1.source = %(source)s") + full_args["source"] = event.source + + # ----- IOS + elif event_type == events.event_type.CLICK_IOS.ui_type: + event_from = event_from % f"{events.event_type.CLICK_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CLICK_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + + elif event_type == events.event_type.INPUT_IOS.ui_type: + event_from = event_from % f"{events.event_type.INPUT_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + if event.custom is not None and len(event.custom) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + value_key="custom{i}")) + full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} + elif event_type == events.event_type.VIEW_IOS.ui_type: + event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.VIEW_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.CUSTOM_IOS.ui_type: + event_from = event_from % f"{events.event_type.CUSTOM_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.CUSTOM_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.REQUEST_IOS.ui_type: + event_from = event_from % f"{events.event_type.REQUEST_IOS.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST_IOS.column} {op} %({e_k})s", + event.value, value_key=e_k)) + elif event_type == events.event_type.ERROR_IOS.ui_type: + event_from = event_from % f"{events.event_type.ERROR_IOS.table} AS main INNER JOIN public.crashes_ios AS main1 USING(crash_id)" + if not is_any and event.value not in [None, "*", ""]: + event_where.append( + _multiple_conditions(f"(main1.reason {op} %({e_k})s OR main1.name {op} %({e_k})s)", + event.value, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.fetch_failed: + event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + event.value, value_key=e_k)) + col = performance_event.get_col(event_type) + colname = col["column"] + event_where.append(f"main.{colname} = FALSE") + # elif event_type == schemas.PerformanceEventType.fetch_duration: + # event_from = event_from % f"{events.event_type.REQUEST.table} AS main " + # if not is_any: + # event_where.append( + # _multiple_conditions(f"main.{events.event_type.REQUEST.column} {op} %({e_k})s", + # event.value, value_key=e_k)) + # col = performance_event.get_col(event_type) + # colname = col["column"] + # tname = "main" + # e_k += "_custom" + # full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + # _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + # event.custom, value_key=e_k)) + elif event_type in [schemas.PerformanceEventType.location_dom_complete, + schemas.PerformanceEventType.location_largest_contentful_paint_time, + schemas.PerformanceEventType.location_ttfb, + schemas.PerformanceEventType.location_avg_cpu_load, + schemas.PerformanceEventType.location_avg_memory_usage + ]: + event_from = event_from % f"{events.event_type.LOCATION.table} AS main " + col = performance_event.get_col(event_type) + colname = col["column"] + tname = "main" + if col.get("extraJoin") is not None: + tname = "ej" + event_from += f" INNER JOIN {col['extraJoin']} AS {tname} USING(session_id)" + event_where += [f"{tname}.timestamp >= main.timestamp", f"{tname}.timestamp >= %(startDate)s", + f"{tname}.timestamp <= %(endDate)s"] + if not is_any: + event_where.append( + _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", + event.value, value_key=e_k)) + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + + event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + + _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) + elif event_type == schemas.PerformanceEventType.time_between_events: + event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " + if not isinstance(event.value[0].value, list): + event.value[0].value = [event.value[0].value] + if not isinstance(event.value[1].value, list): + event.value[1].value = [event.value[1].value] + event.value[0].value = helper.values_for_operator(value=event.value[0].value, + op=event.value[0].operator) + event.value[1].value = helper.values_for_operator(value=event.value[1].value, + op=event.value[0].operator) + e_k1 = e_k + "_e1" + e_k2 = e_k + "_e2" + full_args = {**full_args, + **_multiple_values(event.value[0].value, value_key=e_k1), + **_multiple_values(event.value[1].value, value_key=e_k2)} + s_op = __get_sql_operator(event.value[0].operator) + event_where += ["main2.timestamp >= %(startDate)s", "main2.timestamp <= %(endDate)s"] + if event_index > 0 and not or_events: + event_where.append("main2.session_id=event_0.session_id") + event_where.append( + _multiple_conditions( + f"main.{getattr(events.event_type, event.value[0].type).column} {s_op} %({e_k1})s", + event.value[0].value, value_key=e_k1)) + s_op = __get_sql_operator(event.value[1].operator) + event_where.append( + _multiple_conditions( + f"main2.{getattr(events.event_type, event.value[1].type).column} {s_op} %({e_k2})s", + event.value[1].value, value_key=e_k2)) + + e_k += "_custom" + full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + event_where.append( + _multiple_conditions(f"main2.timestamp - main.timestamp {event.customOperator} %({e_k})s", + event.custom, value_key=e_k)) + + + else: + continue + if event_index == 0 or or_events: + event_where += ss_constraints + if is_not: + if event_index == 0 or or_events: + events_query_from.append(f"""\ + (SELECT + session_id, + 0 AS timestamp + FROM sessions + WHERE EXISTS(SELECT session_id + FROM {event_from} + WHERE {" AND ".join(event_where)} + AND sessions.session_id=ms.session_id) IS FALSE + AND project_id = %(projectId)s + AND start_ts >= %(startDate)s + AND start_ts <= %(endDate)s + AND duration IS NOT NULL + ) {"" if or_events else ("AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + else: + events_query_from.append(f"""\ + (SELECT + event_0.session_id, + event_{event_index - 1}.timestamp AS timestamp + WHERE EXISTS(SELECT session_id FROM {event_from} WHERE {" AND ".join(event_where)}) IS FALSE + ) AS event_{event_index} {"ON(TRUE)" if event_index > 0 else ""}\ + """) + else: + events_query_from.append(f"""\ + (SELECT main.session_id, MIN(main.timestamp) AS timestamp + FROM {event_from} + WHERE {" AND ".join(event_where)} + GROUP BY 1 + ) {"" if or_events else (f"AS event_{event_index} " + ("ON(TRUE)" if event_index > 0 else ""))}\ + """) + event_index += 1 + if event_index > 0: + if or_events: + events_query_part = f"""SELECT + session_id, + MIN(timestamp) AS first_event_ts, + MAX(timestamp) AS last_event_ts + FROM ({events_joiner.join(events_query_from)}) AS u + GROUP BY 1 + {fav_only_join}""" + else: + events_query_part = f"""SELECT + event_0.session_id, + MIN(event_0.timestamp) AS first_event_ts, + MAX(event_{event_index - 1}.timestamp) AS last_event_ts + FROM {events_joiner.join(events_query_from)} + GROUP BY 1 + {fav_only_join}""" + else: + data.events = [] + # --------------------------------------------------------------------------- + if data.startDate is not None: + extra_constraints.append("s.start_ts >= %(startDate)s") + if data.endDate is not None: + extra_constraints.append("s.start_ts <= %(endDate)s") + # if data.platform is not None: + # if data.platform == schemas.PlatformType.mobile: + # extra_constraints.append(b"s.user_os in ('Android','BlackBerry OS','iOS','Tizen','Windows Phone')") + # elif data.platform == schemas.PlatformType.desktop: + # extra_constraints.append( + # b"s.user_os in ('Chrome OS','Fedora','Firefox OS','Linux','Mac OS X','Ubuntu','Windows')") + if data.order is None: + data.order = "DESC" + sort = 'session_id' + if data.sort is not None and data.sort != "session_id": + sort += " " + data.order + "," + helper.key_to_snake_case(data.sort) + else: + sort = 'session_id' + if errors_only: + extra_from += f" INNER JOIN {events.event_type.ERROR.table} AS er USING (session_id) INNER JOIN public.errors AS ser USING (error_id)" + extra_constraints.append("ser.source = 'js_exception'") + if error_status != "ALL": + extra_constraints.append("ser.status = %(error_status)s") + full_args["status"] = error_status.lower() + if favorite_only: + extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" + extra_constraints.append("ufe.user_id = %(user_id)s") + # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] + if not favorite_only and not errors_only: + extra_from += """LEFT JOIN (SELECT user_id, session_id + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" + extra_join = "" + if issue is not None: + extra_join = """ + INNER JOIN LATERAL(SELECT TRUE FROM events_common.issues INNER JOIN public.issues AS p_issues USING (issue_id) + WHERE issues.session_id=f.session_id + AND p_issues.type=%(issue_type)s + AND p_issues.context_string=%(issue_contextString)s + AND timestamp >= f.first_event_ts + AND timestamp <= f.last_event_ts) AS issues ON(TRUE) + """ + full_args["issue_contextString"] = issue["contextString"] + full_args["issue_type"] = issue["type"] + query_part = f"""\ + FROM {f"({events_query_part}) AS f" if len(events_query_part) > 0 else "public.sessions AS s"} + {extra_join} + {"INNER JOIN public.sessions AS s USING(session_id)" if len(events_query_part) > 0 else ""} + {extra_from} + WHERE + {" AND ".join(extra_constraints)}""" + return full_args, query_part, sort + + def search_by_metadata(tenant_id, user_id, m_key, m_value, project_id=None): if project_id is None: all_projects = projects.get_projects(tenant_id=tenant_id, recording_state=False) diff --git a/api/routers/core.py b/api/routers/core.py index a509db750..df80a110a 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -10,7 +10,8 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig log_tool_stackdriver, reset_password, sessions_favorite_viewed, \ log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \ log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github, \ - assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, slack, users + assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, slack, users, \ + custom_metrics from chalicelib.core.collaboration_slack import Slack from chalicelib.utils import email_helper from chalicelib.utils.TimeUTC import TimeUTC @@ -1086,3 +1087,38 @@ def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), return users.change_password(email=context.email, old_password=data.old_password, new_password=data.new_password, tenant_id=context.tenant_id, user_id=context.user_id) + + +@app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) +def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return custom_metrics.try_live(project_id=projectId, data=data) + +@app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) +def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) + + +@app.get('/{projectId}/custom_metrics', tags=["customMetrics"]) +def get_custom_metrics(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get_all(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} + + +@app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.update(user_id=context.user_id, metric_id=metric_id,data=data)} + + +@app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) +def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} diff --git a/api/schemas.py b/api/schemas.py index fe5296e52..edbcdbdcb 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -576,3 +576,33 @@ class SentrySchema(BaseModel): class MobileSignPayloadSchema(BaseModel): keys: List[str] = Field(...) + + +class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): + startDate: Optional[int] = Field(None) + endDate: Optional[int] = Field(None) + sort: Optional[str] = Field(None) + order: Optional[str] = Field(None) + + +class CustomMetricCreateSeriesSchema(BaseModel): + title: Optional[str] = Field(None) + index: Optional[int] = Field(None) + filter: Optional[CustomMetricSeriesFilterSchema] = Field([]) + + +class CreateCustomMetricsSchema(BaseModel): + title: str = Field(...) + series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) + is_public: Optional[bool] = Field(False) + + class Config: + alias_generator = attribute_to_camel_case + + +class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): + series_id: Optional[int] = Field(None) + + +class UpdateCustomMetricsSchema(CreateCustomMetricsSchema): + series: List[CustomMetricUpdateSeriesSchema] = Field(..., min_items=1) diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index a9d7767d0..75efdd452 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -1,5 +1,6 @@ import json +import schemas from chalicelib.core import users from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC @@ -104,8 +105,8 @@ def get_project(tenant_id, project_id, include_last_session=False, include_gdpr= query = cur.mogrify(f"""\ SELECT s.project_id, - s.name, - s.project_key + s.project_key, + s.name {",(SELECT max(ss.start_ts) FROM public.sessions AS ss WHERE ss.project_id = %(project_id)s) AS last_recorded_session_at" if include_last_session else ""} {',s.gdpr' if include_gdpr else ''} {tracker_query} @@ -129,20 +130,20 @@ def is_authorized(project_id, tenant_id): return get_project(tenant_id=tenant_id, project_id=project_id) is not None -def create(tenant_id, user_id, data, skip_authorization=False): +def create(tenant_id, user_id, data: schemas.CreateProjectSchema, skip_authorization=False): if not skip_authorization: admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} - return {"data": __create(tenant_id=tenant_id, name=data.get("name", "my first project"))} + return {"data": __create(tenant_id=tenant_id, name=data.name)} -def edit(tenant_id, user_id, project_id, data): +def edit(tenant_id, user_id, project_id, data: schemas.CreateProjectSchema): admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} return {"data": __update(tenant_id=tenant_id, project_id=project_id, - changes={"name": data.get("name", "my first project")})} + changes={"name": data.name})} def delete(tenant_id, user_id, project_id): From d4a788a73a4c370d8789c3c67ab79e0da6f0e9ce Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 6 Jan 2022 11:02:42 +0100 Subject: [PATCH 047/206] feat(api): retry to create DB-connexion-pool in case of disruption --- api/chalicelib/utils/pg_client.py | 40 ++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index 097ecfafe..ec4694fe1 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -21,20 +21,38 @@ class ORThreadedConnectionPool(psycopg2.pool.ThreadedConnectionPool): def getconn(self, *args, **kwargs): self._semaphore.acquire() - return super().getconn(*args, **kwargs) + try: + return super().getconn(*args, **kwargs) + except psycopg2.pool.PoolError as e: + if str(e) == "connection pool is closed": + make_pool() + raise e def putconn(self, *args, **kwargs): super().putconn(*args, **kwargs) self._semaphore.release() -try: - postgreSQL_pool = ORThreadedConnectionPool(config("pg_minconn", cast=int, default=20), 100, **PG_CONFIG) - if (postgreSQL_pool): - print("Connection pool created successfully") -except (Exception, psycopg2.DatabaseError) as error: - print("Error while connecting to PostgreSQL", error) - raise error +postgreSQL_pool: ORThreadedConnectionPool = None + + +def make_pool(): + global postgreSQL_pool + if postgreSQL_pool is not None: + try: + postgreSQL_pool.closeall() + except (Exception, psycopg2.DatabaseError) as error: + print("Error while closing all connexions to PostgreSQL", error) + try: + postgreSQL_pool = ORThreadedConnectionPool(config("pg_minconn", cast=int, default=20), 100, **PG_CONFIG) + if (postgreSQL_pool): + print("Connection pool created successfully") + except (Exception, psycopg2.DatabaseError) as error: + print("Error while connecting to PostgreSQL", error) + raise error + + +make_pool() class PostgresClient: @@ -62,7 +80,11 @@ class PostgresClient: self.connection.close() except Exception as error: print("Error while committing/closing PG-connection", error) - raise error + if str(error) == "connection already closed": + print("Recreating the connexion pool") + make_pool() + else: + raise error finally: postgreSQL_pool.putconn(self.connection) From a2d9366c9fd913a1b2138d6a9a60fa3ab9825308 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 6 Jan 2022 11:27:51 +0100 Subject: [PATCH 048/206] feat(DB): metrics and metric_series tables --- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 23 +++++++++++++++++++ .../db/init_dbs/postgresql/init_schema.sql | 23 +++++++++++++++++++ .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 23 +++++++++++++++++++ .../db/init_dbs/postgresql/init_schema.sql | 22 ++++++++++++++++++ 4 files changed, 91 insertions(+) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 808bdb9cc..01c2cf46c 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -80,4 +80,27 @@ CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.perfor CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; +CREATE TABLE metrics +( + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + title text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp +); +CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); +CREATE TABLE metric_series +( + series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, + index integer NOT NULL, + title text NULL, + filter jsonb NOT NULL, + created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, + deleted_at timestamp +); +CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); + COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 24d0b6423..593d1966c 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -974,6 +974,29 @@ $$ CREATE INDEX traces_user_id_idx ON traces (user_id); CREATE INDEX traces_tenant_id_idx ON traces (tenant_id); + CREATE TABLE metrics + ( + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + title text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp + ); + CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); + CREATE TABLE metric_series + ( + series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, + index integer NOT NULL, + title text NULL, + filter jsonb NOT NULL, + created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, + deleted_at timestamp + ); + CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); + raise notice 'DB created'; END IF; END; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index a44de93bf..5a9a82b56 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -31,4 +31,27 @@ CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.perfor CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; +CREATE TABLE metrics +( + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + title text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp +); +CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); +CREATE TABLE metric_series +( + series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, + index integer NOT NULL, + title text NULL, + filter jsonb NOT NULL, + created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, + deleted_at timestamp +); +CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); + COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index d40fdab5d..ea8b62612 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -919,6 +919,28 @@ $$ CREATE INDEX jobs_start_at_idx ON jobs (start_at); CREATE INDEX jobs_project_id_idx ON jobs (project_id); + CREATE TABLE metrics + ( + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + title text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp + ); + CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); + CREATE TABLE metric_series + ( + series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, + index integer NOT NULL, + title text NULL, + filter jsonb NOT NULL, + created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, + deleted_at timestamp + ); + CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); raise notice 'DB created'; END IF; From f7cc837944560c55bb05aaadf5082c9e352bd03f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 6 Jan 2022 14:42:48 +0100 Subject: [PATCH 049/206] feat(api): custom metrics preview --- api/chalicelib/core/custom_metrics.py | 34 +++++--------------- api/chalicelib/core/sessions.py | 45 ++++++++++++++++++++++----- api/routers/core.py | 11 ++++--- api/schemas.py | 6 ++++ 4 files changed, 57 insertions(+), 39 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index f716c4799..5f5f6c27e 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -1,37 +1,19 @@ import json import schemas +from chalicelib.core import sessions from chalicelib.utils import helper, pg_client from chalicelib.utils.TimeUTC import TimeUTC -def try_live(project_id,data: schemas.CreateCustomMetricsSchema): - with pg_client.PostgresClient() as cur: - _data = {} - for i, s in enumerate(data.series): - for k in s.dict().keys(): - _data[f"{k}_{i}"] = s.__getattribute__(k) - _data[f"index_{i}"] = i - _data[f"filter_{i}"] = s.filter.json() - series_len = len(data.series) - data.series = None - params = { "project_id": project_id, **data.dict(), **_data} - query = cur.mogrify(f"""\ - WITH m AS (INSERT INTO metrics (project_id, user_id, title) - VALUES (%(project_id)s, %(user_id)s, %(title)s) - RETURNING *) - INSERT - INTO metric_series(metric_id, index, title, filter) - VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(title_{i})s, %(filter_{i})s::jsonb)" - for i in range(series_len)])} - RETURNING metric_id;""", params) +def try_live(project_id, data: schemas.TryCustomMetricsSchema): + results = [] + for s in data.series: + s.filter.startDate = data.startDate + s.filter.endDate = data.endDate + results.append(sessions.search2_series(data=s.filter, project_id=project_id, density=data.density)) + return results - cur.execute( - query - ) - r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return {"data": r} def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): with pg_client.PostgresClient() as cur: diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 9f4213add..8ec2e2afc 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -1,7 +1,7 @@ import schemas from chalicelib.core import events, metadata, events_ios, \ sessions_mobs, issues, projects, errors, resources, assist, performance_event -from chalicelib.utils import pg_client, helper, dev +from chalicelib.utils import pg_client, helper, dev, metrics_helper SESSION_PROJECTION_COLS = """s.project_id, s.session_id::text AS session_id, @@ -199,11 +199,11 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f # ORDER BY favorite DESC, issue_score DESC, {sort} {order};""", # full_args) - print("--------------------") - print(main_query) + # print("--------------------") + # print(main_query) cur.execute(main_query) - print("--------------------") + # print("--------------------") if count_only: return helper.dict_to_camel_case(cur.fetchone()) sessions = cur.fetchone() @@ -230,6 +230,35 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f } +@dev.timed +def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, density: int): + step_size = metrics_helper.__get_step_size(endTimestamp=data.endDate, startTimestamp=data.startDate, + density=density, factor=1) + full_args, query_part, sort = search_query_parts(data=data, error_status=None, errors_only=False, + favorite_only=False, issue=None, project_id=project_id, + user_id=None) + full_args["step_size"] = step_size + with pg_client.PostgresClient() as cur: + main_query = cur.mogrify(f"""WITH full_sessions AS (SELECT DISTINCT ON(s.session_id) s.session_id, s.start_ts + {query_part}) + SELECT generated_timestamp AS timestamp, + COUNT(s) AS count + FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT 1 AS s + FROM full_sessions + WHERE start_ts >= generated_timestamp + AND start_ts < generated_timestamp + %(step_size)s) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""", full_args) + + # print("--------------------") + # print(main_query) + cur.execute(main_query) + # print("--------------------") + sessions = cur.fetchall() + return sessions + + def search_query_parts(data, error_status, errors_only, favorite_only, issue, project_id, user_id): ss_constraints = [] full_args = {"project_id": project_id, "startDate": data.startDate, "endDate": data.endDate, @@ -707,11 +736,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr extra_from += " INNER JOIN public.user_favorite_errors AS ufe USING (error_id)" extra_constraints.append("ufe.user_id = %(user_id)s") # extra_constraints = [extra.decode('UTF-8') + "\n" for extra in extra_constraints] - if not favorite_only and not errors_only: + if not favorite_only and not errors_only and user_id is not None: extra_from += """LEFT JOIN (SELECT user_id, session_id - FROM public.user_favorite_sessions - WHERE user_id = %(userId)s) AS favorite_sessions - USING (session_id)""" + FROM public.user_favorite_sessions + WHERE user_id = %(userId)s) AS favorite_sessions + USING (session_id)""" extra_join = "" if issue is not None: extra_join = """ diff --git a/api/routers/core.py b/api/routers/core.py index df80a110a..eb81c32ce 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1091,14 +1091,15 @@ def change_client_password(data: schemas.EditUserPasswordSchema = Body(...), @app.post('/{projectId}/custom_metrics/try', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics/try', tags=["customMetrics"]) -def try_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): - return custom_metrics.try_live(project_id=projectId, data=data) +def try_custom_metric(projectId: int, data: schemas.TryCustomMetricsSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.try_live(project_id=projectId, data=data)} + @app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), - context: schemas.CurrentContext = Depends(OR_context)): + context: schemas.CurrentContext = Depends(OR_context)): return custom_metrics.create(project_id=projectId, user_id=context.user_id, data=data) @@ -1116,7 +1117,7 @@ def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentCo @app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.update(user_id=context.user_id, metric_id=metric_id,data=data)} + return {"data": custom_metrics.update(user_id=context.user_id, metric_id=metric_id, data=data)} @app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) diff --git a/api/schemas.py b/api/schemas.py index edbcdbdcb..864b27425 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -600,6 +600,12 @@ class CreateCustomMetricsSchema(BaseModel): alias_generator = attribute_to_camel_case +class TryCustomMetricsSchema(CreateCustomMetricsSchema): + startDate: int = Field(TimeUTC.now(-7)) + endDate: int = Field(TimeUTC.now()) + density: int = Field(7) + + class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): series_id: Optional[int] = Field(None) From 74694eb5b445d7aa776c8b03d4e9191227eb1b26 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 6 Jan 2022 17:15:12 +0100 Subject: [PATCH 050/206] feat(api): custom metrics preview view type (line&progress) --- api/chalicelib/core/custom_metrics.py | 12 +++++++++- api/chalicelib/core/sessions.py | 34 +++++++++++++++++---------- api/schemas.py | 6 +++++ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 5f5f6c27e..dc405b090 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -11,7 +11,17 @@ def try_live(project_id, data: schemas.TryCustomMetricsSchema): for s in data.series: s.filter.startDate = data.startDate s.filter.endDate = data.endDate - results.append(sessions.search2_series(data=s.filter, project_id=project_id, density=data.density)) + results.append(sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, + view_type=data.view_type)) + if data.view_type == schemas.MetricViewType.progress: + r = {"count": results[-1]} + diff = s.filter.endDate - s.filter.startDate + s.filter.startDate = data.endDate + s.filter.endDate = data.endDate - diff + r["previousCount"] = sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, + view_type=data.view_type) + r["countProgress"] = helper.__progress(old_val=r["previousCount"], new_val=r["count"]) + results[-1] = r return results diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8ec2e2afc..8eb58a29d 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -231,7 +231,8 @@ def search2_pg(data: schemas.SessionsSearchPayloadSchema, project_id, user_id, f @dev.timed -def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, density: int): +def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, density: int, + view_type: schemas.MetricViewType): step_size = metrics_helper.__get_step_size(endTimestamp=data.endDate, startTimestamp=data.startDate, density=density, factor=1) full_args, query_part, sort = search_query_parts(data=data, error_status=None, errors_only=False, @@ -239,23 +240,30 @@ def search2_series(data: schemas.SessionsSearchPayloadSchema, project_id: int, d user_id=None) full_args["step_size"] = step_size with pg_client.PostgresClient() as cur: - main_query = cur.mogrify(f"""WITH full_sessions AS (SELECT DISTINCT ON(s.session_id) s.session_id, s.start_ts - {query_part}) - SELECT generated_timestamp AS timestamp, - COUNT(s) AS count - FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp - LEFT JOIN LATERAL ( SELECT 1 AS s - FROM full_sessions - WHERE start_ts >= generated_timestamp - AND start_ts < generated_timestamp + %(step_size)s) AS sessions ON (TRUE) - GROUP BY generated_timestamp - ORDER BY generated_timestamp;""", full_args) + if view_type == schemas.MetricViewType.line_chart: + main_query = cur.mogrify(f"""WITH full_sessions AS (SELECT DISTINCT ON(s.session_id) s.session_id, s.start_ts + {query_part}) + SELECT generated_timestamp AS timestamp, + COUNT(s) AS count + FROM generate_series(%(startDate)s, %(endDate)s, %(step_size)s) AS generated_timestamp + LEFT JOIN LATERAL ( SELECT 1 AS s + FROM full_sessions + WHERE start_ts >= generated_timestamp + AND start_ts < generated_timestamp + %(step_size)s) AS sessions ON (TRUE) + GROUP BY generated_timestamp + ORDER BY generated_timestamp;""", full_args) + else: + main_query = cur.mogrify(f"""SELECT count(DISTINCT s.session_id) AS count + {query_part};""", full_args) # print("--------------------") # print(main_query) cur.execute(main_query) # print("--------------------") - sessions = cur.fetchall() + if view_type == schemas.MetricViewType.line_chart: + sessions = cur.fetchall() + else: + sessions = cur.fetchone()["count"] return sessions diff --git a/api/schemas.py b/api/schemas.py index 864b27425..351d826da 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -600,10 +600,16 @@ class CreateCustomMetricsSchema(BaseModel): alias_generator = attribute_to_camel_case +class MetricViewType(str, Enum): + line_chart = "lineChart" + progress = "progress" + + class TryCustomMetricsSchema(CreateCustomMetricsSchema): startDate: int = Field(TimeUTC.now(-7)) endDate: int = Field(TimeUTC.now()) density: int = Field(7) + view_type: MetricViewType = Field(MetricViewType.line_chart) class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): From ad6caaa225f9c5dadafb197e1b21725ecb4341f3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 6 Jan 2022 19:22:21 +0100 Subject: [PATCH 051/206] feat(api): custom metrics rename title to name feat(api): save search feat(api): refactored funnels feat(api): removed funnels EE feat(DB): saved search table feat(DB): custom metrics rename title to name feat(DB): funnels new index --- api/chalicelib/core/custom_metrics.py | 33 ++- api/chalicelib/core/funnels.py | 26 +- api/chalicelib/core/saved_search.py | 115 ++++++++ api/routers/core.py | 39 ++- api/schemas.py | 6 +- ee/api/.gitignore | 1 + ee/api/chalicelib/core/funnels.py | 275 ------------------ .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 21 +- .../db/init_dbs/postgresql/init_schema.sql | 20 +- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 22 +- .../db/init_dbs/postgresql/init_schema.sql | 22 +- 11 files changed, 266 insertions(+), 314 deletions(-) create mode 100644 api/chalicelib/core/saved_search.py delete mode 100644 ee/api/chalicelib/core/funnels.py diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index dc405b090..f7f203fb6 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -37,12 +37,12 @@ def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): data.series = None params = {"user_id": user_id, "project_id": project_id, **data.dict(), **_data} query = cur.mogrify(f"""\ - WITH m AS (INSERT INTO metrics (project_id, user_id, title) - VALUES (%(project_id)s, %(user_id)s, %(title)s) + WITH m AS (INSERT INTO metrics (project_id, user_id, name) + VALUES (%(project_id)s, %(user_id)s, %(name)s) RETURNING *) INSERT - INTO metric_series(metric_id, index, title, filter) - VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(title_{i})s, %(filter_{i})s::jsonb)" + INTO metric_series(metric_id, index, name, filter) + VALUES {",".join([f"((SELECT metric_id FROM m), %(index_{i})s, %(name_{i})s, %(filter_{i})s::jsonb)" for i in range(series_len)])} RETURNING metric_id;""", params) @@ -69,13 +69,14 @@ def __get_series_id(metric_id): return [r["series_id"] for r in rows] -def update(metric_id, user_id, data: schemas.UpdateCustomMetricsSchema): +def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSchema): series_ids = __get_series_id(metric_id) n_series = [] d_series_ids = [] u_series = [] u_series_ids = [] - params = {"metric_id": metric_id, "is_public": data.is_public, "title": data.title} + params = {"metric_id": metric_id, "is_public": data.is_public, "name": data.name, + "user_id": user_id, "project_id": project_id} for i, s in enumerate(data.series): prefix = "u_" if s.series_id is None: @@ -98,18 +99,18 @@ def update(metric_id, user_id, data: schemas.UpdateCustomMetricsSchema): sub_queries = [] if len(n_series) > 0: sub_queries.append(f"""\ - n AS (INSERT INTO metric_series (metric_id, index, title, filter) - VALUES {",".join([f"(%(metric_id)s, %(n_index_{s['i']})s, %(n_title_{s['i']})s, %(n_filter_{s['i']})s::jsonb)" + n AS (INSERT INTO metric_series (metric_id, index, name, filter) + VALUES {",".join([f"(%(metric_id)s, %(n_index_{s['i']})s, %(n_name_{s['i']})s, %(n_filter_{s['i']})s::jsonb)" for s in n_series])} RETURNING 1)""") if len(u_series) > 0: sub_queries.append(f"""\ u AS (UPDATE metric_series - SET title=series.title, + SET name=series.name, filter=series.filter, index=series.filter.index - FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_title_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" - for s in n_series])}) AS series(series_id, index, title, filter) + FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_name_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" + for s in n_series])}) AS series(series_id, index, name, filter) WHERE metric_id =%(metric_id)s AND series_id=series.series_id RETURNING 1)""") if len(d_series_ids) > 0: @@ -119,7 +120,10 @@ def update(metric_id, user_id, data: schemas.UpdateCustomMetricsSchema): query = cur.mogrify(f"""\ {"WITH " if len(sub_queries) > 0 else ""}{",".join(sub_queries)} UPDATE metrics - SET title = %(title)s, is_public= %(is_public)s WHERE metric_id = %(metric_id)s + SET name = %(name)s, is_public= %(is_public)s + WHERE metric_id = %(metric_id)s + AND project_id = %(project_id)s + AND (user_id = %(user_id)s OR is_public) RETURNING metric_id;""", params) cur.execute( query @@ -161,7 +165,8 @@ def delete(project_id, metric_id, user_id): UPDATE public.metrics SET deleted_at = timezone('utc'::text, now()) WHERE project_id = %(project_id)s - AND metric_id = %(metric_id)s;""", + AND metric_id = %(metric_id)s + AND (user_id = %(user_id)s OR is_public);""", {"metric_id": metric_id, "project_id": project_id, "user_id": user_id}) ) @@ -188,5 +193,5 @@ def get(metric_id, project_id, user_id): ) ) row = cur.fetchone() - row["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 7d0bcee7d..d73204c77 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -40,7 +40,7 @@ def create(project_id, user_id, name, filter, is_public): return {"data": r} -def update(funnel_id, user_id, name=None, filter=None, is_public=None): +def update(funnel_id, user_id, project_id, name=None, filter=None, is_public=None): s_query = [] if filter is not None: helper.delete_keys_from_dict(filter, REMOVE_KEYS) @@ -56,9 +56,10 @@ def update(funnel_id, user_id, name=None, filter=None, is_public=None): UPDATE public.funnels SET {" , ".join(s_query)} WHERE funnel_id=%(funnel_id)s - RETURNING *;""", - {"user_id": user_id, "funnel_id": funnel_id, "name": name, - "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) + AND project_id = %(project_id)s + AND (user_id = %(user_id)s OR is_public) + RETURNING *;""", {"user_id": user_id, "funnel_id": funnel_id, "name": name, + "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) # print("--------------------") # print(query) # print("--------------------") @@ -74,13 +75,12 @@ def update(funnel_id, user_id, name=None, filter=None, is_public=None): def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date=None, details=False): with pg_client.PostgresClient() as cur: - team_query = "" cur.execute( cur.mogrify( f"""\ - SELECT DISTINCT ON (funnels.funnel_id) funnel_id,project_id, user_id, name, created_at, deleted_at, is_public + SELECT funnel_id, project_id, user_id, name, created_at, deleted_at, is_public {",filter" if details else ""} - FROM public.funnels {team_query} + FROM public.funnels WHERE project_id = %(project_id)s AND funnels.deleted_at IS NULL AND (funnels.user_id = %(user_id)s OR funnels.is_public);""", @@ -135,7 +135,8 @@ def delete(project_id, funnel_id, user_id): UPDATE public.funnels SET deleted_at = timezone('utc'::text, now()) WHERE project_id = %(project_id)s - AND funnel_id = %(funnel_id)s;""", + AND funnel_id = %(funnel_id)s + AND (user_id = %(user_id)s OR is_public);""", {"funnel_id": funnel_id, "project_id": project_id, "user_id": user_id}) ) @@ -220,7 +221,7 @@ def get_issues_on_the_fly(funnel_id, project_id, data): last_stage=last_stage))} -def get(funnel_id, project_id): +def get(funnel_id, project_id, user_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -230,8 +231,9 @@ def get(funnel_id, project_id): FROM public.funnels WHERE project_id = %(project_id)s AND deleted_at IS NULL - AND funnel_id = %(funnel_id)s;""", - {"funnel_id": funnel_id, "project_id": project_id} + AND funnel_id = %(funnel_id)s + AND (user_id = %(user_id)s OR is_public);""", + {"funnel_id": funnel_id, "project_id": project_id, "user_id": user_id} ) ) @@ -247,7 +249,7 @@ def get(funnel_id, project_id): @dev.timed def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value=None, start_date=None, end_date=None): if len(data.get("events", [])) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.get('startDate', start_date), diff --git a/api/chalicelib/core/saved_search.py b/api/chalicelib/core/saved_search.py new file mode 100644 index 000000000..dfa9a1dcf --- /dev/null +++ b/api/chalicelib/core/saved_search.py @@ -0,0 +1,115 @@ +import json + +import schemas +from chalicelib.utils import helper, pg_client +from chalicelib.utils.TimeUTC import TimeUTC + + +def create(project_id, user_id, data: schemas.SavedSearchSchema): + with pg_client.PostgresClient() as cur: + data = data.dict() + data["filter"] = json.dumps(data["filter"]) + query = cur.mogrify("""\ + INSERT INTO public.searches (project_id, user_id, name, filter,is_public) + VALUES (%(project_id)s, %(user_id)s, %(name)s, %(filter)s::jsonb,%(is_public)s) + RETURNING *;""", {"user_id": user_id, "project_id": project_id, **data}) + cur.execute( + query + ) + r = cur.fetchone() + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r = helper.dict_to_camel_case(r) + return {"data": r} + + +def update(search_id, project_id, user_id, data: schemas.SavedSearchSchema): + with pg_client.PostgresClient() as cur: + data = data.dict() + data["filter"] = json.dumps(data["filter"]) + query = cur.mogrify(f"""\ + UPDATE public.searches + SET name = %(name)s, + filter = %(filter)s, + is_public = %(is_public)s + WHERE search_id=%(search_id)s + AND project_id= %(project_id)s + AND (user_id = %(user_id)s OR is_public) + RETURNING *;""", {"search_id": search_id, "project_id": project_id, "user_id": user_id, **data}) + cur.execute( + query + ) + r = cur.fetchone() + r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r = helper.dict_to_camel_case(r) + # r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) + return r + + +def get_all(project_id, user_id, details=False): + with pg_client.PostgresClient() as cur: + print(cur.mogrify( + f"""\ + SELECT search_id, project_id, user_id, name, created_at, deleted_at, is_public + {",filter" if details else ""} + FROM public.searches + WHERE project_id = %(project_id)s + AND deleted_at IS NULL + AND (user_id = %(user_id)s OR is_public);""", + {"project_id": project_id, "user_id": user_id} + )) + cur.execute( + cur.mogrify( + f"""\ + SELECT search_id, project_id, user_id, name, created_at, deleted_at, is_public + {",filter" if details else ""} + FROM public.searches + WHERE project_id = %(project_id)s + AND deleted_at IS NULL + AND (user_id = %(user_id)s OR is_public);""", + {"project_id": project_id, "user_id": user_id} + ) + ) + + rows = cur.fetchall() + rows = helper.list_to_camel_case(rows) + for row in rows: + row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) + return rows + + +def delete(project_id, search_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify("""\ + UPDATE public.searches + SET deleted_at = timezone('utc'::text, now()) + WHERE project_id = %(project_id)s + AND search_id = %(search_id)s + AND (user_id = %(user_id)s OR is_public);""", + {"search_id": search_id, "project_id": project_id, "user_id": user_id}) + ) + + return {"state": "success"} + + +def get(search_id, project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT + * + FROM public.searches + WHERE project_id = %(project_id)s + AND deleted_at IS NULL + AND search_id = %(search_id)s + AND (user_id = %(user_id)s OR is_public);""", + {"search_id": search_id, "project_id": project_id, "user_id": user_id} + ) + ) + + f = helper.dict_to_camel_case(cur.fetchone()) + if f is None: + return None + + f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) + return f diff --git a/api/routers/core.py b/api/routers/core.py index eb81c32ce..c2099779d 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -11,7 +11,7 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \ log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github, \ assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, slack, users, \ - custom_metrics + custom_metrics, saved_search from chalicelib.core.collaboration_slack import Slack from chalicelib.utils import email_helper from chalicelib.utils.TimeUTC import TimeUTC @@ -752,7 +752,7 @@ def get_funnel_issue_sessions(projectId: int, funnelId: int, issueId: str, @app.get('/{projectId}/funnels/{funnelId}', tags=["funnels"]) def get_funnel(projectId: int, funnelId: int, context: schemas.CurrentContext = Depends(OR_context)): - data = funnels.get(funnel_id=funnelId, project_id=projectId) + data = funnels.get(funnel_id=funnelId, project_id=projectId, user_id=context.user_id) if data is None: return {"errors": ["funnel not found"]} return {"data": data} @@ -766,7 +766,8 @@ def edit_funnel(projectId: int, funnelId: int, data: schemas.UpdateFunnelSchema user_id=context.user_id, name=data.name, filter=data.filter.dict(), - is_public=data.is_public) + is_public=data.is_public, + project_id=projectId) @app.delete('/{projectId}/funnels/{funnelId}', tags=["funnels"]) @@ -1117,9 +1118,39 @@ def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentCo @app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.update(user_id=context.user_id, metric_id=metric_id, data=data)} + return { + "data": custom_metrics.update(project_id=projectId, user_id=context.user_id, metric_id=metric_id, data=data)} @app.delete('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) def delete_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": custom_metrics.delete(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} + + +@app.post('/{projectId}/saved_search', tags=["savedSearch"]) +@app.put('/{projectId}/saved_search', tags=["savedSearch"]) +def add_saved_search(projectId: int, data: schemas.SavedSearchSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return saved_search.create(project_id=projectId, user_id=context.user_id, data=data) + + +@app.get('/{projectId}/saved_search', tags=["savedSearch"]) +def get_saved_searches(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": saved_search.get_all(project_id=projectId, user_id=context.user_id)} + + +@app.get('/{projectId}/saved_search/{search_id}', tags=["savedSearch"]) +def get_saved_search(projectId: int, search_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": saved_search.get(project_id=projectId, search_id=search_id, user_id=context.user_id)} + + +@app.post('/{projectId}/saved_search/{search_id}', tags=["savedSearch"]) +@app.put('/{projectId}/saved_search/{search_id}', tags=["savedSearch"]) +def update_saved_search(projectId: int, search_id: int, data: schemas.SavedSearchSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": saved_search.update(user_id=context.user_id, search_id=search_id, data=data, project_id=projectId)} + + +@app.delete('/{projectId}/saved_search/{search_id}', tags=["savedSearch"]) +def delete_saved_search(projectId: int, search_id: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": saved_search.delete(project_id=projectId, user_id=context.user_id, search_id=search_id)} diff --git a/api/schemas.py b/api/schemas.py index 351d826da..35983175d 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -592,7 +592,7 @@ class CustomMetricCreateSeriesSchema(BaseModel): class CreateCustomMetricsSchema(BaseModel): - title: str = Field(...) + name: str = Field(...) series: List[CustomMetricCreateSeriesSchema] = Field(..., min_items=1) is_public: Optional[bool] = Field(False) @@ -618,3 +618,7 @@ class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): class UpdateCustomMetricsSchema(CreateCustomMetricsSchema): series: List[CustomMetricUpdateSeriesSchema] = Field(..., min_items=1) + + +class SavedSearchSchema(FunnelSchema): + pass diff --git a/ee/api/.gitignore b/ee/api/.gitignore index fbf1958af..0cfd228b2 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -186,6 +186,7 @@ Pipfile /chalicelib/core/errors_favorite_viewed.py /chalicelib/core/events.py /chalicelib/core/events_ios.py +/chalicelib/core/funnels.py /chalicelib/core/integration_base.py /chalicelib/core/integration_base_issue.py /chalicelib/core/integration_github.py diff --git a/ee/api/chalicelib/core/funnels.py b/ee/api/chalicelib/core/funnels.py deleted file mode 100644 index 9c550244d..000000000 --- a/ee/api/chalicelib/core/funnels.py +++ /dev/null @@ -1,275 +0,0 @@ -import chalicelib.utils.helper -from chalicelib.core import events, significance, sessions -from chalicelib.utils.TimeUTC import TimeUTC - -from chalicelib.utils import helper, pg_client -from chalicelib.utils import dev -import json - -REMOVE_KEYS = ["key", "_key", "startDate", "endDate"] - -ALLOW_UPDATE_FOR = ["name", "filter"] - - -def filter_stages(stages): - ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, - events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, - events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, - events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] - return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] - - -def create(project_id, user_id, name, filter, is_public): - helper.delete_keys_from_dict(filter, REMOVE_KEYS) - filter["events"] = filter_stages(stages=filter.get("events", [])) - with pg_client.PostgresClient() as cur: - query = cur.mogrify("""\ - INSERT INTO public.funnels (project_id, user_id, name, filter,is_public) - VALUES (%(project_id)s, %(user_id)s, %(name)s, %(filter)s::jsonb,%(is_public)s) - RETURNING *;""", - {"user_id": user_id, "project_id": project_id, "name": name, "filter": json.dumps(filter), - "is_public": is_public}) - - cur.execute( - query - ) - r = cur.fetchone() - r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) - r = helper.dict_to_camel_case(r) - r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) - return {"data": r} - - -def update(funnel_id, user_id, name=None, filter=None, is_public=None): - s_query = [] - if filter is not None: - helper.delete_keys_from_dict(filter, REMOVE_KEYS) - s_query.append("filter = %(filter)s::jsonb") - if name is not None and len(name) > 0: - s_query.append("name = %(name)s") - if is_public is not None: - s_query.append("is_public = %(is_public)s") - if len(s_query) == 0: - return {"errors": ["Nothing to update"]} - with pg_client.PostgresClient() as cur: - query = cur.mogrify(f"""\ - UPDATE public.funnels - SET {" , ".join(s_query)} - WHERE funnel_id=%(funnel_id)s - RETURNING *;""", - {"user_id": user_id, "funnel_id": funnel_id, "name": name, - "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) - # print("--------------------") - # print(query) - # print("--------------------") - cur.execute( - query - ) - r = cur.fetchone() - r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) - r = helper.dict_to_camel_case(r) - r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) - return {"data": r} - - -def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date=None, details=False): - with pg_client.PostgresClient() as cur: - team_query = """INNER JOIN - ( - SELECT collaborators.user_id - FROM public.users AS creator - INNER JOIN public.users AS collaborators USING (tenant_id) - WHERE creator.user_id=%(user_id)s - ) AS team USING (user_id)""" - cur.execute( - cur.mogrify( - f"""\ - SELECT DISTINCT ON (funnels.funnel_id) funnel_id,project_id, user_id, name, created_at, deleted_at, is_public - {",filter" if details else ""} - FROM public.funnels {team_query} - WHERE project_id = %(project_id)s - AND funnels.deleted_at IS NULL - AND (funnels.user_id = %(user_id)s OR funnels.is_public);""", - {"project_id": project_id, "user_id": user_id} - ) - ) - - rows = cur.fetchall() - rows = helper.list_to_camel_case(rows) - for row in rows: - row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) - if details: - row["filter"]["events"] = filter_stages(row["filter"]["events"]) - get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, - end_date=end_date) - counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) - row["sessionsCount"] = counts["countSessions"] - row["usersCount"] = counts["countUsers"] - overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) - row["stages"] = overview["stages"] - row.pop("filter") - row["stagesCount"] = len(row["stages"]) - # TODO: ask david to count it alone - row["criticalIssuesCount"] = overview["criticalIssuesCount"] - row["missedConversions"] = 0 if len(row["stages"]) < 2 \ - else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] - return rows - - -def get_possible_issue_types(project_id): - return [{"type": t, "title": chalicelib.utils.helper.get_issue_title(t)} for t in - ['click_rage', 'dead_click', 'excessive_scrolling', - 'bad_request', 'missing_resource', 'memory', 'cpu', - 'slow_resource', 'slow_page_load', 'crash', 'custom_event_error', - 'js_error']] - - -def get_start_end_time(filter_d, range_value, start_date, end_date): - if start_date is not None and end_date is not None: - filter_d["startDate"], filter_d["endDate"] = start_date, end_date - elif range_value is not None and len(range_value) > 0: - filter_d["rangeValue"] = range_value - filter_d["startDate"], filter_d["endDate"] = TimeUTC.get_start_end_from_range(range_value) - else: - filter_d["startDate"], filter_d["endDate"] = TimeUTC.get_start_end_from_range(filter_d["rangeValue"]) - - -def delete(project_id, funnel_id, user_id): - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify("""\ - UPDATE public.funnels - SET deleted_at = timezone('utc'::text, now()) - WHERE project_id = %(project_id)s - AND funnel_id = %(funnel_id)s;""", - {"funnel_id": funnel_id, "project_id": project_id, "user_id": user_id}) - ) - - return {"data": {"state": "success"}} - - -def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) - - -def get_sessions_on_the_fly(funnel_id, project_id, user_id, data): - data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - return sessions.search2_pg(data=data, project_id=project_id, user_id=user_id) - - -def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=f["filter"], project_id=project_id) - insights[-1]["dropDueToIssues"] = total_drop_due_to_issues - return {"data": {"stages": helper.list_to_camel_case(insights), - "totalDropDueToIssues": total_drop_due_to_issues}} - - -def get_top_insights_on_the_fly(funnel_id, project_id, data): - data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - if len(insights) > 0: - insights[-1]["dropDueToIssues"] = total_drop_due_to_issues - return {"data": {"stages": helper.list_to_camel_case(insights), - "totalDropDueToIssues": total_drop_due_to_issues}} - - -def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - return {"data": { - "issues": helper.dict_to_camel_case(significance.get_issues_list(filter_d=f["filter"], project_id=project_id)) - }} - - -@dev.timed -def get_issues_on_the_fly(funnel_id, project_id, data): - first_stage = data.get("firstStage") - last_stage = data.get("lastStage") - data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - return { - "issues": helper.dict_to_camel_case( - significance.get_issues_list(filter_d=data, project_id=project_id, first_stage=first_stage, - last_stage=last_stage))} - - -def get(funnel_id, project_id): - with pg_client.PostgresClient() as cur: - cur.execute( - cur.mogrify( - """\ - SELECT - * - FROM public.funnels - WHERE project_id = %(project_id)s - AND deleted_at IS NULL - AND funnel_id = %(funnel_id)s;""", - {"funnel_id": funnel_id, "project_id": project_id} - ) - ) - - f = helper.dict_to_camel_case(cur.fetchone()) - if f is None: - return None - - f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) - f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) - return f - - -@dev.timed -def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value=None, start_date=None, end_date=None): - if len(data.get("events", [])) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) - if f is None: - return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.get('startDate', start_date), - end_date=data.get('endDate', end_date)) - data = f["filter"] - - # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - issues = get_issues_on_the_fly(funnel_id=funnel_id, project_id=project_id, data=data).get("issues", {}) - issues = issues.get("significant", []) + issues.get("insignificant", []) - issue = None - for i in issues: - if i.get("issueId", "") == issue_id: - issue = i - break - return {"sessions": sessions.search2_pg(user_id=user_id, project_id=project_id, issue=issue, - data=data) if issue is not None else {"total": 0, "sessions": []}, - # "stages": helper.list_to_camel_case(insights), - # "totalDropDueToIssues": total_drop_due_to_issues, - "issue": issue} diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 01c2cf46c..618b83cb3 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -85,7 +85,7 @@ CREATE TABLE metrics metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - title text NOT NULL, + name text NOT NULL, is_public boolean NOT NULL DEFAULT FALSE, created_at timestamp default timezone('utc'::text, now()) not null, deleted_at timestamp @@ -96,11 +96,28 @@ CREATE TABLE metric_series series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, index integer NOT NULL, - title text NULL, + name text NULL, filter jsonb NOT NULL, created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, deleted_at timestamp ); CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); +CREATE INDEX IF NOT EXISTS funnels_project_id_idx ON public.funnels (project_id); + + +CREATE TABLE searches +( + search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + name text not null, + filter jsonb not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + is_public boolean NOT NULL DEFAULT False +); + +CREATE INDEX IF NOT EXISTS searches_user_id_is_public_idx ON public.searches (user_id, is_public); +CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_id); COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 593d1966c..bc3ccf4f6 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -382,6 +382,7 @@ $$ ); CREATE INDEX funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); + CREATE INDEX funnels_project_id_idx ON public.funnels (project_id); -- --- announcements.sql --- @@ -979,7 +980,7 @@ $$ metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - title text NOT NULL, + name text NOT NULL, is_public boolean NOT NULL DEFAULT FALSE, created_at timestamp default timezone('utc'::text, now()) not null, deleted_at timestamp @@ -990,13 +991,28 @@ $$ series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, index integer NOT NULL, - title text NULL, + name text NULL, filter jsonb NOT NULL, created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, deleted_at timestamp ); CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); + CREATE TABLE searches + ( + search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + name text not null, + filter jsonb not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + is_public boolean NOT NULL DEFAULT False + ); + + CREATE INDEX searches_user_id_is_public_idx ON public.searches (user_id, is_public); + CREATE INDEX searches_project_id_idx ON public.searches (project_id); + raise notice 'DB created'; END IF; END; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 5a9a82b56..d23b4ef29 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -36,7 +36,7 @@ CREATE TABLE metrics metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - title text NOT NULL, + name text NOT NULL, is_public boolean NOT NULL DEFAULT FALSE, created_at timestamp default timezone('utc'::text, now()) not null, deleted_at timestamp @@ -47,11 +47,29 @@ CREATE TABLE metric_series series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, index integer NOT NULL, - title text NULL, + name text NULL, filter jsonb NOT NULL, created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, deleted_at timestamp ); CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); +CREATE INDEX IF NOT EXISTS funnels_project_id_idx ON public.funnels (project_id); + + +CREATE TABLE searches +( + search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + name text not null, + filter jsonb not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + is_public boolean NOT NULL DEFAULT False +); + +CREATE INDEX IF NOT EXISTS searches_user_id_is_public_idx ON public.searches (user_id, is_public); +CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_id); + COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index ea8b62612..3c06ae840 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -349,6 +349,7 @@ $$ ); CREATE INDEX funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); + CREATE INDEX funnels_project_id_idx ON public.funnels (project_id); -- --- announcements.sql --- @@ -924,7 +925,7 @@ $$ metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - title text NOT NULL, + name text NOT NULL, is_public boolean NOT NULL DEFAULT FALSE, created_at timestamp default timezone('utc'::text, now()) not null, deleted_at timestamp @@ -935,13 +936,30 @@ $$ series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, index integer NOT NULL, - title text NULL, + name text NULL, filter jsonb NOT NULL, created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, deleted_at timestamp ); CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); + + CREATE TABLE searches + ( + search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + name text not null, + filter jsonb not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + is_public boolean NOT NULL DEFAULT False + ); + + CREATE INDEX searches_user_id_is_public_idx ON public.searches (user_id, is_public); + CREATE INDEX searches_project_id_idx ON public.searches (project_id); + + raise notice 'DB created'; END IF; END; From 6b58a93fd5f94b9b70d9f9fe58f48d006bb6fce4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 7 Jan 2022 12:42:03 +0100 Subject: [PATCH 052/206] feat(api): custom metrics get chart --- api/chalicelib/core/custom_metrics.py | 12 +++++++++--- api/routers/core.py | 7 +++++++ api/schemas.py | 13 ++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index f7f203fb6..7d20fd39b 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -12,19 +12,25 @@ def try_live(project_id, data: schemas.TryCustomMetricsSchema): s.filter.startDate = data.startDate s.filter.endDate = data.endDate results.append(sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, - view_type=data.view_type)) - if data.view_type == schemas.MetricViewType.progress: + view_type=data.viewType)) + if data.viewType == schemas.MetricViewType.progress: r = {"count": results[-1]} diff = s.filter.endDate - s.filter.startDate s.filter.startDate = data.endDate s.filter.endDate = data.endDate - diff r["previousCount"] = sessions.search2_series(data=s.filter, project_id=project_id, density=data.density, - view_type=data.view_type) + view_type=data.viewType) r["countProgress"] = helper.__progress(old_val=r["previousCount"], new_val=r["count"]) results[-1] = r return results +def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id) + metric: schemas.TryCustomMetricsSchema = schemas.TryCustomMetricsSchema.parse_obj({**data.dict(), **metric}) + return try_live(project_id=project_id, data=metric) + + def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): with pg_client.PostgresClient() as cur: _data = {} diff --git a/api/routers/core.py b/api/routers/core.py index c2099779d..cf76140f7 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1114,6 +1114,13 @@ def get_custom_metric(projectId: int, metric_id: int, context: schemas.CurrentCo return {"data": custom_metrics.get(project_id=projectId, user_id=context.user_id, metric_id=metric_id)} +@app.post('/{projectId}/custom_metrics/{metric_id}/chart', tags=["customMetrics"]) +def get_custom_metric_chart(projectId: int, metric_id: int, data: schemas.CustomMetricChartPayloadSchema = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.make_chart(project_id=projectId, user_id=context.user_id, metric_id=metric_id, + data=data)} + + @app.post('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics/{metric_id}', tags=["customMetrics"]) def update_custom_metric(projectId: int, metric_id: int, data: schemas.UpdateCustomMetricsSchema = Body(...), diff --git a/api/schemas.py b/api/schemas.py index 35983175d..d783241e7 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -586,7 +586,7 @@ class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): class CustomMetricCreateSeriesSchema(BaseModel): - title: Optional[str] = Field(None) + name: Optional[str] = Field(None) index: Optional[int] = Field(None) filter: Optional[CustomMetricSeriesFilterSchema] = Field([]) @@ -605,11 +605,18 @@ class MetricViewType(str, Enum): progress = "progress" -class TryCustomMetricsSchema(CreateCustomMetricsSchema): +class CustomMetricChartPayloadSchema(BaseModel): startDate: int = Field(TimeUTC.now(-7)) endDate: int = Field(TimeUTC.now()) density: int = Field(7) - view_type: MetricViewType = Field(MetricViewType.line_chart) + viewType: MetricViewType = Field(MetricViewType.line_chart) + + class Config: + alias_generator = attribute_to_camel_case + + +class TryCustomMetricsSchema(CreateCustomMetricsSchema, CustomMetricChartPayloadSchema): + name: Optional[str] = Field(None) class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): From 55a86d76f21bea8ec360082717ebf18e0f27a04d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 7 Jan 2022 19:52:54 +0100 Subject: [PATCH 053/206] feat(api): new alerts detection for threshold only feat(api): old alerts detection for threshold only feat(DB): alerts changes feat(DB): alerts indexes --- api/chalicelib/core/alerts.py | 105 +++++---- api/chalicelib/core/alerts_processor.py | 208 ++++++++++++++++++ api/routers/core.py | 4 +- api/schemas.py | 25 ++- ee/api/.gitignore | 3 + .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 5 + .../db/init_dbs/postgresql/init_schema.sql | 54 +++-- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 6 + .../db/init_dbs/postgresql/init_schema.sql | 5 +- 9 files changed, 331 insertions(+), 84 deletions(-) create mode 100644 api/chalicelib/core/alerts_processor.py diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index 4c8bb151d..4fc628408 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -3,11 +3,13 @@ import time from fastapi import BackgroundTasks +import schemas from chalicelib.core import notifications, slack, webhook from chalicelib.utils import pg_client, helper, email_helper from chalicelib.utils.TimeUTC import TimeUTC -ALLOW_UPDATE = ["name", "description", "active", "detectionMethod", "query", "options"] + +# ALLOW_UPDATE = ["name", "description", "active", "detectionMethod", "query", "options"] def get(id): @@ -38,32 +40,32 @@ def get_all(project_id): return all -SUPPORTED_THRESHOLD = [15, 30, 60, 120, 240, 1440] - - -def __transform_structure(data): - if data.get("options") is None: - return f"Missing 'options'", None - if data["options"].get("currentPeriod") not in SUPPORTED_THRESHOLD: - return f"Unsupported currentPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None - if data["options"].get("previousPeriod", 15) not in SUPPORTED_THRESHOLD: - return f"Unsupported previousPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None - if data["options"].get("renotifyInterval") is None: - data["options"]["renotifyInterval"] = 720 - data["query"]["right"] = float(data["query"]["right"]) - data["query"] = json.dumps(data["query"]) - data["description"] = data["description"] if data.get("description") is not None and len( - data["description"]) > 0 else None - if data.get("options"): - messages = [] - for m in data["options"].get("message", []): - if m.get("value") is None: - continue - m["value"] = str(m["value"]) - messages.append(m) - data["options"]["message"] = messages - data["options"] = json.dumps(data["options"]) - return None, data +# SUPPORTED_THRESHOLD = [15, 30, 60, 120, 240, 1440] +# +# +# def __transform_structure(data): +# if data.get("options") is None: +# return f"Missing 'options'", None +# if data["options"].get("currentPeriod") not in SUPPORTED_THRESHOLD: +# return f"Unsupported currentPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None +# if data["options"].get("previousPeriod", 15) not in SUPPORTED_THRESHOLD: +# return f"Unsupported previousPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None +# if data["options"].get("renotifyInterval") is None: +# data["options"]["renotifyInterval"] = 720 +# data["query"]["right"] = float(data["query"]["right"]) +# data["query"] = json.dumps(data["query"]) +# data["description"] = data["description"] if data.get("description") is not None and len( +# data["description"]) > 0 else None +# if data.get("options"): +# messages = [] +# for m in data["options"].get("message", []): +# if m.get("value") is None: +# continue +# m["value"] = str(m["value"]) +# messages.append(m) +# data["options"]["message"] = messages +# data["options"] = json.dumps(data["options"]) +# return None, data def __process_circular(alert): @@ -74,15 +76,16 @@ def __process_circular(alert): return alert -def create(project_id, data): - err, data = __transform_structure(data) - if err is not None: - return {"errors": [err]} +def create(project_id, data: schemas.AlertSchema): + data = data.dict() + data["query"] = json.dumps(data["query"]) + data["options"] = json.dumps(data["options"]) + with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify("""\ - INSERT INTO public.alerts(project_id, name, description, detection_method, query, options) - VALUES (%(project_id)s, %(name)s, %(description)s, %(detectionMethod)s, %(query)s, %(options)s::jsonb) + INSERT INTO public.alerts(project_id, name, description, detection_method, query, options, series_id) + VALUES (%(project_id)s, %(name)s, %(description)s, %(detectionMethod)s, %(query)s, %(options)s::jsonb, %(series_id)s) RETURNING *;""", {"project_id": project_id, **data}) ) @@ -90,23 +93,33 @@ def create(project_id, data): return {"data": helper.dict_to_camel_case(__process_circular(a))} -def update(id, changes): - changes = {k: changes[k] for k in changes.keys() if k in ALLOW_UPDATE} - err, changes = __transform_structure(changes) - if err is not None: - return {"errors": [err]} - updateq = [] - for k in changes.keys(): - updateq.append(f"{helper.key_to_snake_case(k)} = %({k})s") - if len(updateq) == 0: - return {"errors": ["nothing to update"]} +def update(id, data: schemas.AlertSchema): + data = data.dict() + data["query"] = json.dumps(data["query"]) + data["options"] = json.dumps(data["options"]) + + # changes = {k: changes[k] for k in changes.keys() if k in ALLOW_UPDATE} + # err, changes = __transform_structure(changes) + # if err is not None: + # return {"errors": [err]} + # updateq = [] + # for k in changes.keys(): + # updateq.append(f"{helper.key_to_snake_case(k)} = %({k})s") + # if len(updateq) == 0: + # return {"errors": ["nothing to update"]} with pg_client.PostgresClient() as cur: - query = cur.mogrify(f"""\ + query = cur.mogrify("""\ UPDATE public.alerts - SET {", ".join(updateq)} + SET name = %(name)s, + description = %(description)s, + active = %(active)s, + detection_method = %(detection_method)s, + query = %(query)s, + options = %(options)s, + series_id = %(series_id)s WHERE alert_id =%(id)s AND deleted_at ISNULL RETURNING *;""", - {"id": id, **changes}) + {"id": id, **data}) cur.execute(query=query) a = helper.dict_to_camel_case(cur.fetchone()) return {"data": __process_circular(a)} diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py new file mode 100644 index 000000000..746a6feb5 --- /dev/null +++ b/api/chalicelib/core/alerts_processor.py @@ -0,0 +1,208 @@ +import schemas +from chalicelib.utils import pg_client, helper +from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.core import sessions + +LeftToDb = { + schemas.AlertColumn.performance__dom_content_loaded__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "COALESCE(AVG(NULLIF(dom_content_loaded_time ,0)),0)"}, + schemas.AlertColumn.performance__first_meaningful_paint__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)"}, + schemas.AlertColumn.performance__page_load_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(load_time ,0))"}, + schemas.AlertColumn.performance__dom_build_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(dom_building_time,0))"}, + schemas.AlertColumn.performance__speed_index__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(speed_index,0))"}, + schemas.AlertColumn.performance__page_response_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(response_time,0))"}, + schemas.AlertColumn.performance__ttfb__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(first_paint_time,0))"}, + schemas.AlertColumn.performance__time_to_render__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(visually_complete,0))"}, + schemas.AlertColumn.performance__image_load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='img'"}, + schemas.AlertColumn.performance__request_load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='fetch'"}, + schemas.AlertColumn.resources__load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))"}, + schemas.AlertColumn.resources__missing__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE"}, + schemas.AlertColumn.errors__4xx_5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", + "condition": "status/100!=2"}, + schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__javascript__impacted_sessions__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, + schemas.AlertColumn.performance__crashes__count: { + "table": "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", + "formula": "COUNT(DISTINCT session_id)", "condition": "errors_count > 0"}, + schemas.AlertColumn.errors__javascript__count: { + "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'"}, + schemas.AlertColumn.errors__backend__count: { + "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "source!='js_exception'"}, +} + +# This is the frequency of execution for each threshold +TimeInterval = { + 15: 3, + 30: 5, + 60: 10, + 120: 20, + 240: 30, + 1440: 60, +} + + +def can_check(a) -> bool: + now = TimeUTC.now() + + repetitionBase = a["options"]["currentPeriod"] \ + if a["detectionMethod"] == schemas.AlertDetectionMethod.change \ + and a["options"]["currentPeriod"] > a["options"]["previousPeriod"] \ + else a["options"]["previousPeriod"] + + if TimeInterval.get(repetitionBase) is None: + print(f"repetitionBase: {repetitionBase} NOT FOUND") + return False + + return (a["options"]["renotifyInterval"] <= 0 or + a["options"].get("lastNotification") is None or + a["options"]["lastNotification"] <= 0 or + ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ + and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 + + +def Build(a): + params = {"project_id": a["projectId"]} + full_args={} + if a["seriesId"] is not None: + a["filter"]["sort"]="session_id" + a["filter"]["order"]="DESC" + a["filter"]["startDate"]=-1 + a["filter"]["endDate"]=TimeUTC.now() + full_args, query_part, sort = sessions.search_query_parts(data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), + error_status=None, errors_only=False, + favorite_only=False, issue=None, project_id=a["projectId"], + user_id=None) + subQ=f"""SELECT COUNT(session_id) AS value + {query_part}""" + else: + colDef = LeftToDb[a["query"]["left"]] + subQ = f"""SELECT {colDef["formula"]} AS value + FROM {colDef["table"]} + WHERE project_id = %(project_id)s + {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + # q = sq.Select(fmt.Sprint("value, coalesce(value,0)", a.Query.Operator, a.Query.Right, " AS valid")) + q = f"""SELECT value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" + + # if len(colDef.group) > 0 { + # subQ = subQ.Column(colDef.group + " AS group_value") + # subQ = subQ.GroupBy(colDef.group) + # q = q.Column("group_value") + # } + + if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: + if a["seriesId"]is not None: + q += f""" FROM ({subQ}) AS stat""" + else: + q += f""" FROM ({subQ} AND timestamp>=%(startDate)s + AND sessions.start_ts>=%(startDate)s) AS stat""" + params = {**params, **full_args,"startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} + else: + pass + # if a.Options.Change == "change" : + # if len(colDef.group) == 0 : + # pass + # # sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)).ToSql() + # # sub2, args2, _ := subQ.Where( + # # sq.And{ + # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), + # # }).ToSql() + # # sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")-(" + sub2 + ")) AS value").ToSql() + # # q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) + # else: + # pass + # # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) + # # sub2, args2, _ := subQ.Where( + # # sq.And{ + # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), + # # }).ToSql() + # # sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) + # # q = q.FromSelect(sub1, "stat") + # + # elif a.Options.Change == "percent": + # # if len(colDef.group) == 0 { + # # sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)).ToSql() + # # sub2, args2, _ := subQ.Where( + # # sq.And{ + # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), + # # }).ToSql() + # # sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")/(" + sub2 + ")-1)*100 AS value").ToSql() + # # q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) + # # } else { + # + # pass + # # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) + # # sub2, args2, _ := subQ.Where( + # # sq.And{ + # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), + # # }).ToSql() + # # sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) + # # q = q.FromSelect(sub1, "stat") + # else: + # return q, errors.New("unsupported change method") + + return q, params + + +def process(): + with pg_client.PostgresClient(long_query=True) as cur: + query = """SELECT alert_id, + project_id, + detection_method, + query, + options, + (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, + alerts.name, + alerts.series_id, + filter + FROM public.alerts + LEFT JOIN metric_series USING (series_id) + INNER JOIN projects USING (project_id) + WHERE alerts.deleted_at ISNULL + AND alerts.active + AND projects.active + AND projects.deleted_at ISNULL + AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) + AND alert_id=36 + ORDER BY alerts.created_at;""" + cur.execute(query=query) + all_alerts = helper.list_to_camel_case(cur.fetchall()) + for alert in all_alerts: + if True or can_check(alert): + print(f"Querying alertId:{alert['alertId']} name: {alert['name']}") + query, params = Build(alert) + print(cur.mogrify(query, params)) + print("----------------------") + diff --git a/api/routers/core.py b/api/routers/core.py index cf76140f7..53c8a81c9 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -613,7 +613,7 @@ def errors_merge(context: schemas.CurrentContext = Depends(OR_context)): @app.put('/{projectId}/alerts', tags=["alerts"]) def create_alert(projectId: int, data: schemas.AlertSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return alerts.create(projectId, data.dict()) + return alerts.create(projectId, data) @app.get('/{projectId}/alerts', tags=["alerts"]) @@ -630,7 +630,7 @@ def get_alert(projectId: int, alertId: int, context: schemas.CurrentContext = De @app.put('/{projectId}/alerts/{alertId}', tags=["alerts"]) def update_alert(projectId: int, alertId: int, data: schemas.AlertSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return alerts.update(alertId, data.dict()) + return alerts.update(alertId, data) @app.delete('/{projectId}/alerts/{alertId}', tags=["alerts"]) diff --git a/api/schemas.py b/api/schemas.py index d783241e7..969934f27 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional, List, Union +from typing import Optional, List, Union, Literal from pydantic import BaseModel, Field, EmailStr, HttpUrl, root_validator @@ -278,8 +278,8 @@ class _AlertMessageSchema(BaseModel): class _AlertOptionSchema(BaseModel): message: List[_AlertMessageSchema] = Field([]) - currentPeriod: int = Field(...) - previousPeriod: int = Field(...) + currentPeriod: Literal[15, 30, 60, 120, 240, 1440] = Field(...) + previousPeriod: Literal[15, 30, 60, 120, 240, 1440] = Field(15) lastNotification: Optional[int] = Field(None) renotifyInterval: Optional[int] = Field(720) @@ -304,6 +304,7 @@ class AlertColumn(str, Enum): performance__crashes__count = "performance.crashes.count" errors__javascript__count = "errors.javascript.count" errors__backend__count = "errors.backend.count" + custom = "CUSTOM" class MathOperator(str, Enum): @@ -321,12 +322,24 @@ class _AlertQuerySchema(BaseModel): operator: MathOperator = Field(...) +class AlertDetectionMethod(str, Enum): + threshold = "threshold" + change = "change" + + class AlertSchema(BaseModel): name: str = Field(...) - detectionMethod: str = Field(...) + detectionMethod: AlertDetectionMethod = Field(...) description: Optional[str] = Field(None) options: _AlertOptionSchema = Field(...) query: _AlertQuerySchema = Field(...) + series_id: Optional[int] = Field(None) + + @root_validator + def alert_validator(cls, values): + if values.get("query") is not None and values["query"].left == AlertColumn.custom: + assert values.get("series_id") is not None, "series_id should not be null for CUSTOM alert" + return values class SourcemapUploadPayloadSchema(BaseModel): @@ -451,7 +464,7 @@ class _SessionSearchEventRaw(BaseModel): source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) @root_validator - def check_card_number_omitted(cls, values): + def event_validator(cls, values): if isinstance(values.get("type"), PerformanceEventType): if values.get("type") == PerformanceEventType.fetch_failed: return values @@ -484,7 +497,7 @@ class _SessionSearchFilterSchema(BaseModel): source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) @root_validator - def check_card_number_omitted(cls, values): + def filter_validator(cls, values): if values.get("type") == FilterType.issue: for v in values.get("value"): assert isinstance(v, IssueType), f"value should be of type IssueType for {values.get('type')} filter" diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 0cfd228b2..ee1d296de 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -258,3 +258,6 @@ Pipfile /schemas.py /chalicelib/blueprints/app/v1_api.py /routers/app/v1_api.py +/chalicelib/core/custom_metrics.py +/chalicelib/core/performance_event.py +/chalicelib/core/saved_search.py diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 618b83cb3..c9038c4ae 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -119,5 +119,10 @@ CREATE TABLE searches CREATE INDEX IF NOT EXISTS searches_user_id_is_public_idx ON public.searches (user_id, is_public); CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_id); +CREATE INDEX IF NOT EXISTS alerts_project_id_idx ON alerts (project_id); +ALTER TABLE alerts + ADD COLUMN series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; + +CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index bc3ccf4f6..3138e8d19 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -287,34 +287,6 @@ $$ FOR EACH ROW EXECUTE PROCEDURE notify_project(); --- --- alerts.sql --- - - CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); - - CREATE TABLE alerts - ( - alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - name text NOT NULL, - description text NULL DEFAULT NULL, - active boolean NOT NULL DEFAULT TRUE, - detection_method alert_detection_method NOT NULL, - query jsonb NOT NULL, - deleted_at timestamp NULL DEFAULT NULL, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - options jsonb NOT NULL DEFAULT '{ - "renotifyInterval": 1440 - }'::jsonb - ); - - - CREATE TRIGGER on_insert_or_update_or_delete - AFTER INSERT OR UPDATE OR DELETE - ON alerts - FOR EACH ROW - EXECUTE PROCEDURE notify_alert(); - - -- --- webhooks.sql --- create type webhook_type as enum ('webhook', 'slack', 'email'); @@ -1013,6 +985,32 @@ $$ CREATE INDEX searches_user_id_is_public_idx ON public.searches (user_id, is_public); CREATE INDEX searches_project_id_idx ON public.searches (project_id); + CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); + + CREATE TABLE alerts + ( + alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE, + name text NOT NULL, + description text NULL DEFAULT NULL, + active boolean NOT NULL DEFAULT TRUE, + detection_method alert_detection_method NOT NULL, + query jsonb NOT NULL, + deleted_at timestamp NULL DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + options jsonb NOT NULL DEFAULT '{ + "renotifyInterval": 1440 + }'::jsonb + ); + CREATE INDEX alerts_project_id_idx ON alerts (project_id); + CREATE INDEX alerts_project_id_idx ON alerts (series_id); + CREATE TRIGGER on_insert_or_update_or_delete + AFTER INSERT OR UPDATE OR DELETE + ON alerts + FOR EACH ROW + EXECUTE PROCEDURE notify_alert(); + raise notice 'DB created'; END IF; END; diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index d23b4ef29..42393bb47 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -71,5 +71,11 @@ CREATE TABLE searches CREATE INDEX IF NOT EXISTS searches_user_id_is_public_idx ON public.searches (user_id, is_public); CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_id); +CREATE INDEX IF NOT EXISTS alerts_project_id_idx ON alerts (project_id); + +ALTER TABLE alerts + ADD COLUMN series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; + +CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 3c06ae840..8f74e721e 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -271,6 +271,7 @@ $$ ( alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE, name text NOT NULL, description text NULL DEFAULT NULL, active boolean NOT NULL DEFAULT TRUE, @@ -282,8 +283,8 @@ $$ "renotifyInterval": 1440 }'::jsonb ); - - + CREATE INDEX alerts_project_id_idx ON alerts (project_id); + CREATE INDEX alerts_series_id_idx ON alerts (series_id); CREATE TRIGGER on_insert_or_update_or_delete AFTER INSERT OR UPDATE OR DELETE ON alerts From 6b14f13e53074da05cdea8b08a4b0b40142240ea Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 8 Jan 2022 15:08:12 +0100 Subject: [PATCH 054/206] feat(api): new alerts detection for change(change&percent) feat(api): old alerts detection for change(change&percent) feat(api): custom metrics new endpoints feat(api): custom metrics return series list for alerts feat(DB): alerts-change-change fix --- api/chalicelib/core/alerts_processor.py | 154 ++++++++++-------- api/chalicelib/core/custom_metrics.py | 22 +++ api/chalicelib/utils/pg_client.py | 3 +- api/routers/core.py | 13 ++ api/schemas.py | 21 +++ .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 4 + .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 5 +- 7 files changed, 154 insertions(+), 68 deletions(-) diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 746a6feb5..d811624f8 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -1,7 +1,7 @@ import schemas +from chalicelib.core import sessions from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -from chalicelib.core import sessions LeftToDb = { schemas.AlertColumn.performance__dom_content_loaded__average: { @@ -53,10 +53,10 @@ LeftToDb = { "formula": "COUNT(DISTINCT session_id)", "condition": "errors_count > 0"}, schemas.AlertColumn.errors__javascript__count: { "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'"}, + "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'", "joinSessions": False}, schemas.AlertColumn.errors__backend__count: { "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "source!='js_exception'"}, + "formula": "COUNT(DISTINCT session_id)", "condition": "source!='js_exception'", "joinSessions": False}, } # This is the frequency of execution for each threshold @@ -91,17 +91,19 @@ def can_check(a) -> bool: def Build(a): params = {"project_id": a["projectId"]} - full_args={} + full_args = {} + j_s = True if a["seriesId"] is not None: - a["filter"]["sort"]="session_id" - a["filter"]["order"]="DESC" - a["filter"]["startDate"]=-1 - a["filter"]["endDate"]=TimeUTC.now() - full_args, query_part, sort = sessions.search_query_parts(data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), - error_status=None, errors_only=False, - favorite_only=False, issue=None, project_id=a["projectId"], - user_id=None) - subQ=f"""SELECT COUNT(session_id) AS value + a["filter"]["sort"] = "session_id" + a["filter"]["order"] = "DESC" + a["filter"]["startDate"] = -1 + a["filter"]["endDate"] = TimeUTC.now() + full_args, query_part, sort = sessions.search_query_parts( + data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), + error_status=None, errors_only=False, + favorite_only=False, issue=None, project_id=a["projectId"], + user_id=None) + subQ = f"""SELECT COUNT(session_id) AS value {query_part}""" else: colDef = LeftToDb[a["query"]["left"]] @@ -109,6 +111,9 @@ def Build(a): FROM {colDef["table"]} WHERE project_id = %(project_id)s {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + j_s = colDef.get("joinSessions", True) + print(">>>>>>>>>>>>>") + print(j_s) # q = sq.Select(fmt.Sprint("value, coalesce(value,0)", a.Query.Operator, a.Query.Right, " AS valid")) q = f"""SELECT value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" @@ -119,59 +124,73 @@ def Build(a): # } if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: - if a["seriesId"]is not None: + if a["seriesId"] is not None: q += f""" FROM ({subQ}) AS stat""" else: q += f""" FROM ({subQ} AND timestamp>=%(startDate)s - AND sessions.start_ts>=%(startDate)s) AS stat""" - params = {**params, **full_args,"startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}) AS stat""" + params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} else: - pass - # if a.Options.Change == "change" : - # if len(colDef.group) == 0 : - # pass - # # sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)).ToSql() - # # sub2, args2, _ := subQ.Where( - # # sq.And{ - # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), - # # }).ToSql() - # # sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")-(" + sub2 + ")) AS value").ToSql() - # # q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) - # else: - # pass - # # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) - # # sub2, args2, _ := subQ.Where( - # # sq.And{ - # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), - # # }).ToSql() - # # sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - # # q = q.FromSelect(sub1, "stat") - # - # elif a.Options.Change == "percent": - # # if len(colDef.group) == 0 { - # # sub1, args1, _ := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)).ToSql() - # # sub2, args2, _ := subQ.Where( - # # sq.And{ - # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), - # # }).ToSql() - # # sub1, _, _ = sq.Expr("SELECT ((" + sub1 + ")/(" + sub2 + ")-1)*100 AS value").ToSql() - # # q = q.JoinClause("FROM ("+sub1+") AS stat", append(args1, args2...)...) - # # } else { - # - # pass - # # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) - # # sub2, args2, _ := subQ.Where( - # # sq.And{ - # # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), - # # }).ToSql() - # # sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - # # q = q.FromSelect(sub1, "stat") - # else: - # return q, errors.New("unsupported change method") + if a["options"]["change"] == schemas.AlertDetectionChangeType.change: + # if len(colDef.group) > 0: + # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) + # sub2, args2, _ := subQ.Where( + # sq.And{ + # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), + # }).ToSql() + # sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) + # q = q.FromSelect(sub1, "stat") + # else: + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})-({sub2})) AS value" + q += f" FROM ( {sub1} ) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} + else: + sub1 = f"""{subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND timestamp<%(startDate)s + AND timestamp>=%(timestamp_sub2)s + {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" + params["timestamp_sub2"] = TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000 + sub1 = f"SELECT (( {sub1} )-( {sub2} )) AS value" + q += f" FROM ( {sub1} ) AS stat" + + else: + # if len(colDef.group) >0 { + # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) + # sub2, args2, _ := subQ.Where( + # sq.And{ + # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), + # }).ToSql() + # sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) + # q = q.FromSelect(sub1, "stat") + # } else { + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) \ + * 60 * 1000} + else: + sub1 = f"""{subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND timestamp<%(startDate)s + AND timestamp>=%(timestamp_sub2)s + {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" + params["timestamp_sub2"] = TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) * 60 * 1000 + sub1 = f"SELECT (({sub1})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" return q, params @@ -195,7 +214,6 @@ def process(): AND projects.active AND projects.deleted_at ISNULL AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) - AND alert_id=36 ORDER BY alerts.created_at;""" cur.execute(query=query) all_alerts = helper.list_to_camel_case(cur.fetchall()) @@ -203,6 +221,10 @@ def process(): if True or can_check(alert): print(f"Querying alertId:{alert['alertId']} name: {alert['name']}") query, params = Build(alert) - print(cur.mogrify(query, params)) - print("----------------------") - + query = cur.mogrify(query, params) + # print(alert) + # print(query) + cur.execute(query) + result = cur.fetchone() + if result["valid"]: + print("Valid alert, notifying users") diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 7d20fd39b..ffd911fea 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -201,3 +201,25 @@ def get(metric_id, project_id, user_id): row = cur.fetchone() row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) return helper.dict_to_camel_case(row) + + +def get_series_for_alert(project_id, user_id): + with pg_client.PostgresClient() as cur: + cur.execute( + cur.mogrify( + """SELECT metric_id, + series_id, + metrics.name AS metric_name, + metric_series.name AS series_name, + index AS series_index + FROM metric_series + INNER JOIN metrics USING (metric_id) + WHERE metrics.deleted_at ISNULL + AND metrics.project_id = %(project_id)s + AND (user_id = %(user_id)s OR is_public) + ORDER BY metric_name, series_index, series_name;""", + {"project_id": project_id, "user_id": user_id} + ) + ) + rows = cur.fetchall() + return helper.list_to_camel_case(rows) diff --git a/api/chalicelib/utils/pg_client.py b/api/chalicelib/utils/pg_client.py index ec4694fe1..c598d8971 100644 --- a/api/chalicelib/utils/pg_client.py +++ b/api/chalicelib/utils/pg_client.py @@ -86,7 +86,8 @@ class PostgresClient: else: raise error finally: - postgreSQL_pool.putconn(self.connection) + if not self.long_query: + postgreSQL_pool.putconn(self.connection) def close(): diff --git a/api/routers/core.py b/api/routers/core.py index 53c8a81c9..327cb2eb6 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1097,6 +1097,19 @@ def try_custom_metric(projectId: int, data: schemas.TryCustomMetricsSchema = Bod return {"data": custom_metrics.try_live(project_id=projectId, data=data)} +@app.post('/{projectId}/custom_metrics/chart', tags=["customMetrics"]) +@app.put('/{projectId}/custom_metrics/chart', tags=["customMetrics"]) +def get_custom_metric_chart(projectId: int, data: schemas.CustomMetricChartPayloadSchema2 = Body(...), + context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.make_chart(project_id=projectId, user_id=context.user_id, metric_id=data.metric_id, + data=data)} + + +@app.get('/{projectId}/custom_metrics/series', tags=["customMetrics"]) +def get_series_for_alert(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} + + @app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), diff --git a/api/schemas.py b/api/schemas.py index 969934f27..975ea1eb9 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -276,12 +276,18 @@ class _AlertMessageSchema(BaseModel): value: str = Field(...) +class AlertDetectionChangeType(str, Enum): + percent = "percent" + change = "change" + + class _AlertOptionSchema(BaseModel): message: List[_AlertMessageSchema] = Field([]) currentPeriod: Literal[15, 30, 60, 120, 240, 1440] = Field(...) previousPeriod: Literal[15, 30, 60, 120, 240, 1440] = Field(15) lastNotification: Optional[int] = Field(None) renotifyInterval: Optional[int] = Field(720) + change: Optional[AlertDetectionChangeType] = Field(None) class AlertColumn(str, Enum): @@ -339,8 +345,16 @@ class AlertSchema(BaseModel): def alert_validator(cls, values): if values.get("query") is not None and values["query"].left == AlertColumn.custom: assert values.get("series_id") is not None, "series_id should not be null for CUSTOM alert" + if values.get("detectionMethod") is not None \ + and values["detectionMethod"] == AlertDetectionMethod.change \ + and values.get("options") is not None: + assert values["options"].change is not None, \ + "options.change should not be null for detection method 'change'" return values + class Config: + alias_generator = attribute_to_camel_case + class SourcemapUploadPayloadSchema(BaseModel): urls: List[str] = Field(..., alias="URL") @@ -628,6 +642,10 @@ class CustomMetricChartPayloadSchema(BaseModel): alias_generator = attribute_to_camel_case +class CustomMetricChartPayloadSchema2(CustomMetricChartPayloadSchema): + metric_id: int = Field(...) + + class TryCustomMetricsSchema(CreateCustomMetricsSchema, CustomMetricChartPayloadSchema): name: Optional[str] = Field(None) @@ -635,6 +653,9 @@ class TryCustomMetricsSchema(CreateCustomMetricsSchema, CustomMetricChartPayload class CustomMetricUpdateSeriesSchema(CustomMetricCreateSeriesSchema): series_id: Optional[int] = Field(None) + class Config: + alias_generator = attribute_to_camel_case + class UpdateCustomMetricsSchema(CreateCustomMetricsSchema): series: List[CustomMetricUpdateSeriesSchema] = Field(..., min_items=1) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index c9038c4ae..6cb678c96 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -125,4 +125,8 @@ ALTER TABLE alerts ADD COLUMN series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); +UPDATE alerts +SET options=jsonb_set(options, '{change}', '"change"') +WHERE detection_method = 'change' + AND options -> 'change' ISNULL; COMMIT; \ No newline at end of file diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 42393bb47..3aafca980 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -77,5 +77,8 @@ ALTER TABLE alerts ADD COLUMN series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); - +UPDATE alerts +SET options=jsonb_set(options, '{change}', '"change"') +WHERE detection_method = 'change' + AND options -> 'change' ISNULL; COMMIT; \ No newline at end of file From f98d995906e7116f88890233c6bbd5b21970fcc9 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 8 Jan 2022 17:58:26 +0100 Subject: [PATCH 055/206] feat(api): alerts notifications feat(api): alerts dockerfile feat(api): alerts build script feat(backend): removed alerts --- api/.chalice/config.bundle.json | 67 ---------- api/.chalice/config.json | 68 ---------- api/Dockerfile.alerts | 18 +++ api/app_alerts.py | 26 ++++ api/build.sh | 6 +- api/chalicelib/core/alerts.py | 22 ++- api/chalicelib/core/alerts_processor.py | 57 ++++++-- api/chalicelib/core/webhook.py | 19 ++- api/routers/core.py | 15 +-- api/routers/core_dynamic.py | 9 +- api/schemas.py | 10 -- backend/services/alerts/main.go | 86 ------------ backend/services/alerts/manager.go | 171 ------------------------ ee/api/.chalice/config.json | 81 ----------- ee/api/routers/core_dynamic.py | 9 +- 15 files changed, 122 insertions(+), 542 deletions(-) delete mode 100644 api/.chalice/config.bundle.json delete mode 100644 api/.chalice/config.json create mode 100644 api/Dockerfile.alerts create mode 100644 api/app_alerts.py delete mode 100644 backend/services/alerts/main.go delete mode 100644 backend/services/alerts/manager.go delete mode 100644 ee/api/.chalice/config.json diff --git a/api/.chalice/config.bundle.json b/api/.chalice/config.bundle.json deleted file mode 100644 index 95b29ab50..000000000 --- a/api/.chalice/config.bundle.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "version": "2.0", - "app_name": "parrot", - "environment_variables": { - }, - "stages": { - "default-foss": { - "api_gateway_stage": "default-fos", - "manage_iam_role": false, - "iam_role_arn": "", - "autogen_policy": true, - "environment_variables": { - "isFOS": "true", - "isEE": "false", - "stage": "default-foss", - "jwt_issuer": "openreplay-default-foss", - "sentryURL": "", - "pg_host": "postgresql.db.svc.cluster.local", - "pg_port": "5432", - "pg_dbname": "postgres", - "pg_user": "postgres", - "pg_password": "asayerPostgres", - "alert_ntf": "http://127.0.0.1:8000/async/alerts/notifications/%s", - "email_signup": "http://127.0.0.1:8000/async/email_signup/%s", - "email_funnel": "http://127.0.0.1:8000/async/funnel/%s", - "email_basic": "http://127.0.0.1:8000/async/basic/%s", - "assign_link": "http://127.0.0.1:8000/async/email_assignment", - "captcha_server": "", - "captcha_key": "", - "sessions_bucket": "mobs", - "sessions_region": "us-east-1", - "put_S3_TTL": "20", - "sourcemaps_reader": "http://0.0.0.0:9000/sourcemaps", - "sourcemaps_bucket": "sourcemaps", - "js_cache_bucket": "sessions-assets", - "peers": "http://0.0.0.0:9000/assist/peers", - "async_Token": "", - "EMAIL_HOST": "", - "EMAIL_PORT": "587", - "EMAIL_USER": "", - "EMAIL_PASSWORD": "", - "EMAIL_USE_TLS": "true", - "EMAIL_USE_SSL": "false", - "EMAIL_SSL_KEY": "", - "EMAIL_SSL_CERT": "", - "EMAIL_FROM": "OpenReplay", - "SITE_URL": "", - "announcement_url": "", - "jwt_secret": "", - "jwt_algorithm": "HS512", - "jwt_exp_delta_seconds": "2592000", - "S3_HOST": "", - "S3_KEY": "", - "S3_SECRET": "", - "invitation_link": "/api/users/invitation?token=%s", - "change_password_link": "/reset-password?invitation=%s&&pass=%s", - "version_number": "1.3.5" - }, - "lambda_timeout": 150, - "lambda_memory_size": 400, - "subnet_ids": [ - ], - "security_group_ids": [ - ] - } - } -} diff --git a/api/.chalice/config.json b/api/.chalice/config.json deleted file mode 100644 index d1fe6c36c..000000000 --- a/api/.chalice/config.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "version": "2.0", - "app_name": "parrot", - "environment_variables": { - }, - "stages": { - "default-foss": { - "api_gateway_stage": "default-fos", - "manage_iam_role": false, - "iam_role_arn": "", - "autogen_policy": true, - "environment_variables": { - "isFOS": "true", - "isEE": "false", - "stage": "default-foss", - "jwt_issuer": "openreplay-default-foss", - "sentryURL": "", - "pg_host": "postgresql.db.svc.cluster.local", - "pg_port": "5432", - "pg_dbname": "postgres", - "pg_user": "postgres", - "pg_password": "asayerPostgres", - "alert_ntf": "http://127.0.0.1:8000/async/alerts/notifications/%s", - "email_signup": "http://127.0.0.1:8000/async/email_signup/%s", - "email_funnel": "http://127.0.0.1:8000/async/funnel/%s", - "email_basic": "http://127.0.0.1:8000/async/basic/%s", - "assign_link": "http://127.0.0.1:8000/async/email_assignment", - "captcha_server": "", - "captcha_key": "", - "sessions_bucket": "mobs", - "sessions_region": "us-east-1", - "put_S3_TTL": "20", - "sourcemaps_reader": "http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps", - "sourcemaps_bucket": "sourcemaps", - "js_cache_bucket": "sessions-assets", - "peers": "http://utilities-openreplay.app.svc.cluster.local:9000/assist/%s/peers", - "async_Token": "", - "EMAIL_HOST": "", - "EMAIL_PORT": "587", - "EMAIL_USER": "", - "EMAIL_PASSWORD": "", - "EMAIL_USE_TLS": "true", - "EMAIL_USE_SSL": "false", - "EMAIL_SSL_KEY": "", - "EMAIL_SSL_CERT": "", - "EMAIL_FROM": "OpenReplay", - "SITE_URL": "", - "announcement_url": "", - "jwt_secret": "", - "jwt_algorithm": "HS512", - "jwt_exp_delta_seconds": "2592000", - "S3_HOST": "", - "S3_KEY": "", - "S3_SECRET": "", - "invitation_link": "/api/users/invitation?token=%s", - "change_password_link": "/reset-password?invitation=%s&&pass=%s", - "iosBucket": "openreplay-ios-images", - "version_number": "1.3.6" - }, - "lambda_timeout": 150, - "lambda_memory_size": 400, - "subnet_ids": [ - ], - "security_group_ids": [ - ] - } - } -} \ No newline at end of file diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts new file mode 100644 index 000000000..69e2a1184 --- /dev/null +++ b/api/Dockerfile.alerts @@ -0,0 +1,18 @@ +FROM python:3.9.7-slim +LABEL Maintainer="Rajesh Rajendran" +LABEL Maintainer="KRAIEM Taha Yassine" +WORKDIR /work +COPY . . +RUN pip install -r requirements.txt +RUN mv .env.default .env && rm app.py && ren app_alerts.py app.py +ENV pg_minconn 2 + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD ./entrypoint.sh \ No newline at end of file diff --git a/api/app_alerts.py b/api/app_alerts.py new file mode 100644 index 000000000..0cba8986a --- /dev/null +++ b/api/app_alerts.py @@ -0,0 +1,26 @@ +import logging + +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from fastapi import FastAPI + +from chalicelib.core import alerts_processor + +app = FastAPI() +print("============= ALERTS =============") + + +@app.get("/") +async def root(): + return {"status": "Running"} + + +app.schedule = AsyncIOScheduler() +app.schedule.start() +app.schedule.add_job(id="alerts_processor", **{"func": alerts_processor.process, "trigger": "interval", "minutes": 5, + "misfire_grace_time": 20}) + +for job in app.schedule.get_jobs(): + print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + +logging.basicConfig() +logging.getLogger('apscheduler').setLevel(logging.INFO) diff --git a/api/build.sh b/api/build.sh index ee7cec3ab..c0ede73da 100644 --- a/api/build.sh +++ b/api/build.sh @@ -22,15 +22,19 @@ function build_api(){ # Copy enterprise code [[ $1 == "ee" ]] && { cp -rf ../ee/api/* ./ - cp -rf ../ee/api/.chalice/* ./.chalice/ envarg="default-ee" tag="ee-" } docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/api:${git_sha1} . + docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/api:${git_sha1} docker tag ${DOCKER_REPO:-'local'}/api:${git_sha1} ${DOCKER_REPO:-'local'}/api:${tag}latest docker push ${DOCKER_REPO:-'local'}/api:${tag}latest + + docker push ${DOCKER_REPO:-'local'}/alerts:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/alerts:${git_sha1} ${DOCKER_REPO:-'local'}/alerts:${tag}latest + docker push ${DOCKER_REPO:-'local'}/alerts:${tag}latest } } diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index 4fc628408..ef35d48b3 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -1,8 +1,6 @@ import json import time -from fastapi import BackgroundTasks - import schemas from chalicelib.core import notifications, slack, webhook from chalicelib.utils import pg_client, helper, email_helper @@ -125,7 +123,7 @@ def update(id, data: schemas.AlertSchema): return {"data": __process_circular(a)} -def process_notifications(data, background_tasks: BackgroundTasks): +def process_notifications(data): full = {} for n in data: if "message" in n["options"]: @@ -150,11 +148,23 @@ def process_notifications(data, background_tasks: BackgroundTasks): notifications_list = full[t][i:i + BATCH_SIZE] if t == "slack": - background_tasks.add_task(slack.send_batch, notifications_list=notifications_list) + try: + slack.send_batch(notifications_list=notifications_list) + except Exception as e: + print("!!!Error while sending slack notifications batch") + print(str(e)) elif t == "email": - background_tasks.add_task(send_by_email_batch, notifications_list=notifications_list) + try: + send_by_email_batch(notifications_list=notifications_list) + except Exception as e: + print("!!!Error while sending email notifications batch") + print(str(e)) elif t == "webhook": - background_tasks.add_task(webhook.trigger_batch, data_list=notifications_list) + try: + webhook.trigger_batch(data_list=notifications_list) + except Exception as e: + print("!!!Error while sending webhook notifications batch") + print(str(e)) def send_by_email(notification, destination): diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index d811624f8..281064abf 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -1,5 +1,5 @@ import schemas -from chalicelib.core import sessions +from chalicelib.core import sessions, alerts from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC @@ -112,10 +112,8 @@ def Build(a): WHERE project_id = %(project_id)s {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" j_s = colDef.get("joinSessions", True) - print(">>>>>>>>>>>>>") - print(j_s) - # q = sq.Select(fmt.Sprint("value, coalesce(value,0)", a.Query.Operator, a.Query.Right, " AS valid")) - q = f"""SELECT value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" + + q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" # if len(colDef.group) > 0 { # subQ = subQ.Column(colDef.group + " AS group_value") @@ -196,8 +194,10 @@ def Build(a): def process(): + notifications = [] with pg_client.PostgresClient(long_query=True) as cur: - query = """SELECT alert_id, + query = """SELECT -1 AS tenant_id, + alert_id, project_id, detection_method, query, @@ -217,14 +217,49 @@ def process(): ORDER BY alerts.created_at;""" cur.execute(query=query) all_alerts = helper.list_to_camel_case(cur.fetchall()) + + with pg_client.PostgresClient() as cur: for alert in all_alerts: - if True or can_check(alert): + if can_check(alert): print(f"Querying alertId:{alert['alertId']} name: {alert['name']}") query, params = Build(alert) query = cur.mogrify(query, params) # print(alert) # print(query) - cur.execute(query) - result = cur.fetchone() - if result["valid"]: - print("Valid alert, notifying users") + try: + cur.execute(query) + result = cur.fetchone() + if result["valid"]: + print("Valid alert, notifying users") + notifications.append({ + "alertId": alert["alertId"], + "tenantId": alert["tenantId"], + "title": alert["name"], + "description": f"has been triggered, {alert['query']['left']} = {result['value']} ({alert['query']['operator']} {alert['query']['right']}).", + "buttonText": "Check metrics for more details", + "buttonUrl": f"/{alert['projectId']}/metrics", + "imageUrl": None, + "options": {"source": "ALERT", "sourceId": alert["alertId"], + "sourceMeta": alert["detectionMethod"], + "message": alert["options"]["message"], "projectId": alert["projectId"], + "data": {"title": alert["name"], + "limitValue": alert["query"]["right"], "actualValue": result["value"], + "operator": alert["query"]["operator"], + "trigger": alert["query"]["left"], + "alertId": alert["alertId"], + "detectionMethod": alert["detectionMethod"], + "currentPeriod": alert["options"]["currentPeriod"], + "previousPeriod": alert["options"]["previousPeriod"], + "createdAt": TimeUTC.now()}}, + }) + except Exception as e: + print(f"!!!Error while running alert query for alertId:{alert['alertId']}") + print(str(e)) + print(query) + if len(notifications) > 0: + cur.execute( + cur.mogrify(f"""UPDATE public.Alerts + SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb + WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) + if len(notifications) > 0: + alerts.process_notifications(notifications) diff --git a/api/chalicelib/core/webhook.py b/api/chalicelib/core/webhook.py index 653a2b513..254a46c97 100644 --- a/api/chalicelib/core/webhook.py +++ b/api/chalicelib/core/webhook.py @@ -1,6 +1,7 @@ +import requests + from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -import requests def get_by_id(webhook_id): @@ -81,7 +82,7 @@ def update(tenant_id, webhook_id, changes, replace_none=False): SET {','.join(sub_query)} WHERE webhook_id =%(id)s AND deleted_at ISNULL RETURNING webhook_id AS integration_id, webhook_id AS id,*;""", - {"id": webhook_id, **changes})) + {"id": webhook_id, **changes})) cur.execute( cur.mogrify(f"""\ UPDATE public.webhooks @@ -150,22 +151,18 @@ def trigger_batch(data_list): for w in data_list: if w["destination"] not in webhooks_map: webhooks_map[w["destination"]] = get_by_id(webhook_id=w["destination"]) - __trigger(hook=webhooks_map[w["destination"]], data=w["data"]) + if webhooks_map[w["destination"]] is None: + print(f"!!Error webhook not found: webhook_id={w['destination']}") + else: + __trigger(hook=webhooks_map[w["destination"]], data=w["data"]) def __trigger(hook, data): - if hook["type"] == 'webhook': + if hook is not None and hook["type"] == 'webhook': headers = {} if hook["authHeader"] is not None and len(hook["authHeader"]) > 0: headers = {"Authorization": hook["authHeader"]} - # body = { - # "webhookId": hook["id"], - # "createdAt": TimeUTC.now(), - # "event": event, - # "data": data - # } - r = requests.post(url=hook["endpoint"], json=data, headers=headers) if r.status_code != 200: print("=======> webhook: something went wrong") diff --git a/api/routers/core.py b/api/routers/core.py index 327cb2eb6..78170298a 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -10,7 +10,7 @@ from chalicelib.core import log_tool_rollbar, sourcemaps, events, sessions_assig log_tool_stackdriver, reset_password, sessions_favorite_viewed, \ log_tool_cloudwatch, log_tool_sentry, log_tool_sumologic, log_tools, errors, sessions, \ log_tool_newrelic, announcements, log_tool_bugsnag, weekly_report, integration_jira_cloud, integration_github, \ - assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, slack, users, \ + assist, heatmaps, mobile, signup, tenants, errors_favorite_viewed, boarding, notifications, webhook, users, \ custom_metrics, saved_search from chalicelib.core.collaboration_slack import Slack from chalicelib.utils import email_helper @@ -954,19 +954,6 @@ def add_remove_favorite_error(projectId: int, errorId: str, action: str, startDa return {"errors": ["undefined action"]} -@public_app.post('/async/alerts/notifications/{step}', tags=["async", "alerts"]) -@public_app.put('/async/alerts/notifications/{step}', tags=["async", "alerts"]) -def send_alerts_notification_async(step: str, data: schemas.AlertNotificationSchema = Body(...)): - if data.auth != config("async_Token"): - return {"errors": ["missing auth"]} - if step == "slack": - slack.send_batch(notifications_list=data.notifications) - elif step == "email": - alerts.send_by_email_batch(notifications_list=data.notifications) - elif step == "webhook": - webhook.trigger_batch(data_list=data.notifications) - - @app.get('/notifications', tags=['notifications']) def get_notifications(context: schemas.CurrentContext = Depends(OR_context)): return {"data": notifications.get_all(tenant_id=context.tenant_id, user_id=context.user_id)} diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index 0cd1b98da..d61d2ebfb 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -8,7 +8,7 @@ import schemas from chalicelib.core import assist from chalicelib.core import integrations_manager from chalicelib.core import sessions -from chalicelib.core import tenants, users, metadata, projects, license, alerts +from chalicelib.core import tenants, users, metadata, projects, license from chalicelib.core import webhook from chalicelib.core.collaboration_slack import Slack from chalicelib.utils import captcha @@ -209,13 +209,6 @@ def get_current_plan(context: schemas.CurrentContext = Depends(OR_context)): } -@public_app.post('/alerts/notifications', tags=["alerts"]) -@public_app.put('/alerts/notifications', tags=["alerts"]) -def send_alerts_notifications(background_tasks: BackgroundTasks, data: schemas.AlertNotificationSchema = Body(...)): - # TODO: validate token - return {"data": alerts.process_notifications(data.notifications, background_tasks=background_tasks)} - - @public_app.get('/general_stats', tags=["private"], include_in_schema=False) def get_general_stats(): return {"data": {"sessions:": sessions.count_all()}} diff --git a/api/schemas.py b/api/schemas.py index 975ea1eb9..6a33c20a4 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -88,16 +88,6 @@ class SearchErrorsSchema(BaseModel): order: Optional[str] = Field(None) -class EmailNotificationSchema(BaseModel): - notification: str = Field(...) - destination: str = Field(...) - - -class AlertNotificationSchema(BaseModel): - auth: str = Field(...) - notifications: List[EmailNotificationSchema] = Field(...) - - class CreateNotificationSchema(BaseModel): token: str = Field(...) notifications: List = Field(...) diff --git a/backend/services/alerts/main.go b/backend/services/alerts/main.go deleted file mode 100644 index b11d3ae04..000000000 --- a/backend/services/alerts/main.go +++ /dev/null @@ -1,86 +0,0 @@ -package main - -import ( - "database/sql" - "log" - "os" - "os/signal" - "syscall" - "time" - - "openreplay/backend/pkg/db/postgres" - "openreplay/backend/pkg/env" - _ "github.com/lib/pq" -) - -func main() { - log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - POSTGRES_STRING := env.String("POSTGRES_STRING") - NOTIFICATIONS_STRING := env.String("ALERT_NOTIFICATION_STRING") - log.Printf("Notifications: %s \nPG: %s\n", NOTIFICATIONS_STRING, POSTGRES_STRING) - pg := postgres.NewConn(POSTGRES_STRING) - defer pg.Close() - - pgs, err := sql.Open("postgres", POSTGRES_STRING+ "?sslmode=disable") - if err != nil { - log.Fatal(err) - } - defer pgs.Close() - - manager := NewManager(NOTIFICATIONS_STRING, POSTGRES_STRING, pgs, pg) - if err := pg.IterateAlerts(func(a *postgres.Alert, err error) { - if err != nil { - log.Printf("Postgres error: %v\n", err) - return - } - log.Printf("Alert initialization: %+v\n", *a) - //log.Printf("CreatedAt: %s\n", *a.CreatedAt) - err = manager.Update(a) - if err != nil { - log.Printf("Alert parse error: %v | Alert: %+v\n", err, *a) - return - } - }); err != nil { - log.Fatalf("Postgres error: %v\n", err) - } - - listener, err := postgres.NewAlertsListener(POSTGRES_STRING) - if err != nil { - log.Fatalf("Postgres listener error: %v\n", err) - } - defer listener.Close() - - sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) - - tickAlert := time.Tick(1 * time.Minute) - - log.Printf("Alert service started\n") - manager.RequestAll() - //return - for { - select { - case sig := <-sigchan: - log.Printf("Caught signal %v: terminating\n", sig) - listener.Close() - pg.Close() - os.Exit(0) - case <-tickAlert: - log.Printf("Requesting all...%d alerts\n", manager.Length()) - manager.RequestAll() - case iPointer := <-listener.Alerts: - log.Printf("Alert update: %+v\n", *iPointer) - //log.Printf("CreatedAt: %s\n", *iPointer.CreatedAt) - //log.Printf("Notification received for AlertId: %d\n", iPointer.AlertID) - err := manager.Update(iPointer) - if err != nil { - log.Printf("Alert parse error: %+v | Alert: %v\n", err, *iPointer) - } - case err := <-listener.Errors: - log.Printf("listener error: %v\n", err) - if err.Error() == "conn closed" { - panic("Listener conn lost") - } - } - } -} diff --git a/backend/services/alerts/manager.go b/backend/services/alerts/manager.go deleted file mode 100644 index 11ddb9363..000000000 --- a/backend/services/alerts/manager.go +++ /dev/null @@ -1,171 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "log" - "sync" - "sync/atomic" - "time" - - "openreplay/backend/pkg/db/postgres" -) - -const PGParallelLimit = 2 - -var pgCount int64 - -type manager struct { - postgresString string - notificationsUrl string - alertsCache map[uint32]*postgres.Alert - cacheMutex sync.Mutex - pgParallel chan bool - pgs *sql.DB - pg *postgres.Conn - pgMutex sync.Mutex - notifications map[uint32]*postgres.TenantNotification - notificationsGo *sync.WaitGroup - notificationsMutex sync.Mutex -} - -func NewManager(notificationsUrl string, postgresString string, pgs *sql.DB, pg *postgres.Conn) *manager { - return &manager{ - postgresString: postgresString, - notificationsUrl: notificationsUrl, - alertsCache: make(map[uint32]*postgres.Alert), - cacheMutex: sync.Mutex{}, - pgParallel: make(chan bool, PGParallelLimit), - pgs: pgs, - pg: pg, - pgMutex: sync.Mutex{}, - notifications: make(map[uint32]*postgres.TenantNotification), - notificationsGo: &sync.WaitGroup{}, - notificationsMutex: sync.Mutex{}, - } - -} - -func (m *manager) Length() int { - return len(m.alertsCache) -} - -func (m *manager) Update(a *postgres.Alert) error { - m.cacheMutex.Lock() - defer m.cacheMutex.Unlock() - _, exists := m.alertsCache[a.AlertID] - if exists && a.DeletedAt != nil { - log.Println("deleting alert from memory") - delete(m.alertsCache, a.AlertID) - return nil - } else { - m.alertsCache[a.AlertID] = a - } - return nil -} -func (m *manager) processAlert(a *postgres.Alert) { - defer func() { - defer m.notificationsGo.Done() - <-m.pgParallel - }() - if !a.CanCheck() { - log.Printf("cannot check %s", a.Name) - return - } - //log.Printf("checking %+v", a) - log.Printf("quering %s", a.Name) - //--- For stats: - atomic.AddInt64(&pgCount, 1) - q, err := a.Build() - if err != nil { - log.Println(err) - return - } - - rows, err := q.RunWith(m.pgs).Query() - - if err != nil { - log.Println(err) - return - } - defer rows.Close() - - for rows.Next() { - var ( - value sql.NullFloat64 - valid bool - ) - if err := rows.Scan(&value, &valid); err != nil { - log.Println(err) - continue - } - - if valid && value.Valid { - log.Printf("%s: valid", a.Name) - m.notificationsMutex.Lock() - m.notifications[a.AlertID] = &postgres.TenantNotification{ - TenantId: a.TenantId, - Title: a.Name, - Description: fmt.Sprintf("has been triggered, %s = %.0f (%s %.0f).", a.Query.Left, value.Float64, a.Query.Operator, a.Query.Right), - ButtonText: "Check metrics for more details", - ButtonUrl: fmt.Sprintf("/%d/metrics", a.ProjectID), - ImageUrl: nil, - Options: map[string]interface{}{"source": "ALERT", "sourceId": a.AlertID, "sourceMeta": a.DetectionMethod, "message": a.Options.Message, "projectId": a.ProjectID, "data": map[string]interface{}{"title": a.Name, "limitValue": a.Query.Right, "actualValue": value.Float64, "operator": a.Query.Operator, "trigger": a.Query.Left, "alertId": a.AlertID, "detectionMethod": a.DetectionMethod, "currentPeriod": a.Options.CurrentPeriod, "previousPeriod": a.Options.PreviousPeriod, "createdAt": time.Now().Unix() * 1000}}, - } - m.notificationsMutex.Unlock() - } - } - -} -func (m *manager) RequestAll() { - now := time.Now().Unix() - m.cacheMutex.Lock() - for _, a := range m.alertsCache { - m.pgParallel <- true - m.notificationsGo.Add(1) - go m.processAlert(a) - //m.processAlert(a) - } - //log.Println("releasing cache") - m.cacheMutex.Unlock() - //log.Println("waiting for all alerts to finish") - m.notificationsGo.Wait() - log.Printf("done %d PG queries in: %ds", pgCount, time.Now().Unix()-now) - pgCount = 0 - //log.Printf("Processing %d Notifications", len(m.notifications)) - m.notificationsMutex.Lock() - go m.ProcessNotifications(m.notifications) - m.notificationsMutex.Unlock() - m.notifications = make(map[uint32]*postgres.TenantNotification) - //log.Printf("Notifications purged: %d", len(m.notifications)) -} - -func (m *manager) ProcessNotifications(allNotifications map[uint32]*postgres.TenantNotification) { - if len(allNotifications) == 0 { - log.Println("No notifications to process") - return - } - log.Printf("sending %d notifications", len(allNotifications)) - allIds := make([]uint32, 0, len(allNotifications)) - toSend := postgres.Notifications{ - Notifications: []*postgres.TenantNotification{}, - } - for k, n := range allNotifications { - //log.Printf("notification for %d", k) - allIds = append(allIds, k) - toSend.Notifications = append(toSend.Notifications, n) - } - toSend.Send(m.notificationsUrl) - if err := m.pg.SaveLastNotification(allIds); err != nil { - log.Printf("Error saving LastNotification time: %v", err) - if err.Error() == "conn closed" { - m.pg = postgres.NewConn(m.postgresString) - //if err != nil { - // panic(fmt.Sprintf("Postgres renew notifications connection error: %v\n", err)) - //} - if err := m.pg.SaveLastNotification(allIds); err != nil { - panic(fmt.Sprintf("Error saving LastNotification time, suicide: %v", err)) - } - } - } -} diff --git a/ee/api/.chalice/config.json b/ee/api/.chalice/config.json deleted file mode 100644 index db58c76ba..000000000 --- a/ee/api/.chalice/config.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "version": "2.0", - "app_name": "parrot", - "environment_variables": { - }, - "stages": { - "default-ee": { - "api_gateway_stage": "default-ee", - "manage_iam_role": false, - "iam_role_arn": "", - "autogen_policy": true, - "environment_variables": { - "isFOS": "false", - "isEE": "true", - "stage": "default-ee", - "jwt_issuer": "openreplay-default-ee", - "sentryURL": "", - "pg_host": "127.0.0.1", - "pg_port": "9202", - "pg_dbname": "app", - "pg_user": "", - "pg_password": "", - "ch_host": "", - "ch_port": "", - "alert_ntf": "http://127.0.0.1:8000/async/alerts/notifications/%s", - "email_signup": "http://127.0.0.1:8000/async/email_signup/%s", - "email_funnel": "http://127.0.0.1:8000/async/funnel/%s", - "email_plans": "http://127.0.0.1:8000/async/plans/%s", - "email_basic": "http://127.0.0.1:8000/async/basic/%s", - "assign_link": "http://127.0.0.1:8000/async/email_assignment", - "captcha_server": "", - "captcha_key": "", - "sessions_bucket": "mobs", - "sessions_region": "us-east-1", - "put_S3_TTL": "20", - "sourcemaps_reader": "http://utilities-openreplay.app.svc.cluster.local:9000/sourcemaps", - "sourcemaps_bucket": "sourcemaps", - "peers": "http://utilities-openreplay.app.svc.cluster.local:9000/assist/%s/peers", - "js_cache_bucket": "sessions-assets", - "async_Token": "", - "EMAIL_HOST": "", - "EMAIL_PORT": "587", - "EMAIL_USER": "", - "EMAIL_PASSWORD": "", - "EMAIL_USE_TLS": "true", - "EMAIL_USE_SSL": "false", - "EMAIL_SSL_KEY": "", - "EMAIL_SSL_CERT": "", - "EMAIL_FROM": "OpenReplay", - "SITE_URL": "", - "announcement_url": "", - "jwt_secret": "SET A RANDOM STRING HERE", - "jwt_algorithm": "HS512", - "jwt_exp_delta_seconds": "2592000", - "S3_HOST": "", - "S3_KEY": "", - "S3_SECRET": "", - "LICENSE_KEY": "", - "SAML2_MD_URL": "", - "idp_entityId": "", - "idp_sso_url": "", - "idp_x509cert": "", - "idp_sls_url": "", - "idp_name": "", - "sso_exp_delta_seconds": "172800", - "sso_landing": "/login?jwt=%s", - "invitation_link": "/api/users/invitation?token=%s", - "change_password_link": "/reset-password?invitation=%s&&pass=%s", - "iosBucket": "openreplay-ios-images", - "version_number": "1.3.6", - "assist_secret": "" - }, - "lambda_timeout": 150, - "lambda_memory_size": 400, - "subnet_ids": [ - ], - "security_group_ids": [ - ] - } - } -} diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index a887237b3..bab3689ff 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -8,7 +8,7 @@ import schemas import schemas_ee from chalicelib.core import integrations_manager from chalicelib.core import sessions -from chalicelib.core import tenants, users, metadata, projects, license, alerts, assist +from chalicelib.core import tenants, users, metadata, projects, license, assist from chalicelib.core import webhook from chalicelib.core.collaboration_slack import Slack from chalicelib.utils import captcha, SAML2_helper @@ -213,13 +213,6 @@ def get_current_plan(context: schemas.CurrentContext = Depends(OR_context)): } -@public_app.post('/alerts/notifications', tags=["alerts"]) -@public_app.put('/alerts/notifications', tags=["alerts"]) -def send_alerts_notifications(background_tasks: BackgroundTasks, data: schemas.AlertNotificationSchema = Body(...)): - # TODO: validate token - return {"data": alerts.process_notifications(data.notifications, background_tasks=background_tasks)} - - @public_app.get('/general_stats', tags=["private"], include_in_schema=False) def get_general_stats(): return {"data": {"sessions:": sessions.count_all()}} From 0cc23e68e0b06ea48a23f4c6877eaf7621ffc5e1 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 8 Jan 2022 18:06:05 +0100 Subject: [PATCH 056/206] feat(api): alerts changed dockerfile --- api/Dockerfile.alerts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index 69e2a1184..5299a24f5 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -4,7 +4,7 @@ LABEL Maintainer="KRAIEM Taha Yassine" WORKDIR /work COPY . . RUN pip install -r requirements.txt -RUN mv .env.default .env && rm app.py && ren app_alerts.py app.py +RUN mv .env.default .env && rm app.py && mv app_alerts.py app.py ENV pg_minconn 2 # Add Tini From c016e15cb96e831b22378bf5f9701e491fa47bff Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 10:53:19 +0100 Subject: [PATCH 057/206] feat(api): EE-alerts dockerfile --- ee/api/Dockerfile.alerts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 ee/api/Dockerfile.alerts diff --git a/ee/api/Dockerfile.alerts b/ee/api/Dockerfile.alerts new file mode 100644 index 000000000..b2e3bf48e --- /dev/null +++ b/ee/api/Dockerfile.alerts @@ -0,0 +1,19 @@ +FROM python:3.9.7-slim +LABEL Maintainer="Rajesh Rajendran" +LABEL Maintainer="KRAIEM Taha Yassine" +RUN apt-get update && apt-get install -y pkg-config libxmlsec1-dev gcc && rm -rf /var/lib/apt/lists/* +WORKDIR /work +COPY . . +RUN pip install -r requirements.txt +RUN mv .env.default .env && rm app.py && mv app_alerts.py app.py +ENV pg_minconn 2 + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD ./entrypoint.sh From bdb809db704b9b137a6a6d7cf97d1b05a15b61d6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 12:30:05 +0100 Subject: [PATCH 058/206] feat(api): alerts build script --- api/build.sh | 8 ++----- api/build_alerts.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 api/build_alerts.sh diff --git a/api/build.sh b/api/build.sh index c0ede73da..ef8f8fc3d 100644 --- a/api/build.sh +++ b/api/build.sh @@ -26,17 +26,13 @@ function build_api(){ tag="ee-" } docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/api:${git_sha1} . - docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/api:${git_sha1} docker tag ${DOCKER_REPO:-'local'}/api:${git_sha1} ${DOCKER_REPO:-'local'}/api:${tag}latest docker push ${DOCKER_REPO:-'local'}/api:${tag}latest - - docker push ${DOCKER_REPO:-'local'}/alerts:${git_sha1} - docker tag ${DOCKER_REPO:-'local'}/alerts:${git_sha1} ${DOCKER_REPO:-'local'}/alerts:${tag}latest - docker push ${DOCKER_REPO:-'local'}/alerts:${tag}latest - } +} } check_prereq build_api $1 +bash build_alerts.sh \ No newline at end of file diff --git a/api/build_alerts.sh b/api/build_alerts.sh new file mode 100644 index 000000000..74bf72438 --- /dev/null +++ b/api/build_alerts.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Script to build alerts module +# flags to accept: +# envarg: build for enterprise edition. +# Default will be OSS build. + +# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh + +function make_submodule() { + # -- this part was generated by modules_lister.py -- + mkdir alerts + cp -R ./{app_alerts,schemas}.py ./alerts/ + mkdir -p ./alerts/chalicelib/ + cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ + mkdir -p ./alerts/chalicelib/core/ + cp -R ./chalicelib/core/{__init__,alerts_processor,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,assist,events_ios,sessions_mobs,errors,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ + mkdir -p ./alerts/chalicelib/utils/ + cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ + # -- end of generated part + + cp Dockerfile.alerts ./alerts/ + cd alerts +} + +git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} +envarg="default-foss" +check_prereq() { + which docker || { + echo "Docker not installed, please install docker." + exit=1 + } + [[ exit -eq 1 ]] && exit 1 +} + +function build_api(){ + tag="" + # Copy enterprise code + [[ $1 == "ee" ]] && { + cp -rf ../ee/api/* ./ + envarg="default-ee" + tag="ee-" + } + make_submodule + docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/alerts:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/alerts:${git_sha1} ${DOCKER_REPO:-'local'}/alerts:${tag}latest + docker push ${DOCKER_REPO:-'local'}/alerts:${tag}latest + } + rm -rf alerts +} + +check_prereq +build_api $1 From f9164950a0cb0dc1064994829769e1d04f0d12f4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 12:38:49 +0100 Subject: [PATCH 059/206] feat(api): changed build script --- api/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/build.sh b/api/build.sh index ef8f8fc3d..199448da1 100644 --- a/api/build.sh +++ b/api/build.sh @@ -34,5 +34,4 @@ function build_api(){ } check_prereq -build_api $1 -bash build_alerts.sh \ No newline at end of file +build_api $1 \ No newline at end of file From 4ce69581755d4faf8ea2dd0e775830cc98d6be21 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 12:40:43 +0100 Subject: [PATCH 060/206] feat(api): changed alerts_build script --- api/build_alerts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index 74bf72438..733e9b418 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -19,7 +19,7 @@ function make_submodule() { cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part - cp Dockerfile.alerts ./alerts/ + cp -R ./{Dockerfile.alerts,requirements.txt} ./alerts/ cd alerts } From 9cedb18fb2e495d7fdbcb0bd048771ed3c2e7997 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 13:27:09 +0100 Subject: [PATCH 061/206] feat(api): changed alerts_build script --- api/build_alerts.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index 733e9b418..eac215c0f 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -43,13 +43,14 @@ function build_api(){ } make_submodule docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . + cd .. + rm -rf alerts [[ $PUSH_IMAGE -eq 1 ]] && { docker push ${DOCKER_REPO:-'local'}/alerts:${git_sha1} docker tag ${DOCKER_REPO:-'local'}/alerts:${git_sha1} ${DOCKER_REPO:-'local'}/alerts:${tag}latest docker push ${DOCKER_REPO:-'local'}/alerts:${tag}latest } - rm -rf alerts } check_prereq -build_api $1 +build_api $1 \ No newline at end of file From 826b0e5c351033daee4733a6a29f7983729f0762 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 13:29:00 +0100 Subject: [PATCH 062/206] feat(api): changed alerts_build script --- api/build_alerts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index eac215c0f..0b877006e 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -19,8 +19,7 @@ function make_submodule() { cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part - cp -R ./{Dockerfile.alerts,requirements.txt} ./alerts/ - cd alerts + cp -R ./{Dockerfile.alerts,requirements.txt,.env.default} ./alerts/ } git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} @@ -42,6 +41,7 @@ function build_api(){ tag="ee-" } make_submodule + cd alerts docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . cd .. rm -rf alerts From 7104348f9eace8cdb9a949ebc6deeff732cab64c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 13:48:51 +0100 Subject: [PATCH 063/206] feat(api): changed alerts_build script feat(api): changed Dockerfile feat(api): alerts requirements --- api/Dockerfile.alerts | 4 ++-- api/requirements_alerts.txt | 12 ++++++++++++ ee/api/Dockerfile.alerts | 19 ------------------- 3 files changed, 14 insertions(+), 21 deletions(-) create mode 100644 api/requirements_alerts.txt delete mode 100644 ee/api/Dockerfile.alerts diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index 5299a24f5..ff5e7091e 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -3,8 +3,8 @@ LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" WORKDIR /work COPY . . -RUN pip install -r requirements.txt -RUN mv .env.default .env && rm app.py && mv app_alerts.py app.py +RUN pip install -r requirements_alerts.txt +RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 # Add Tini diff --git a/api/requirements_alerts.txt b/api/requirements_alerts.txt new file mode 100644 index 000000000..da4f74530 --- /dev/null +++ b/api/requirements_alerts.txt @@ -0,0 +1,12 @@ +requests==2.26.0 +urllib3==1.26.6 +boto3==1.16.1 +psycopg2-binary==2.8.6 + + + +fastapi==0.70.1 +uvicorn[standard]==0.16.0 +python-decouple==3.5 +pydantic[email]==1.8.2 +apscheduler==3.8.1 \ No newline at end of file diff --git a/ee/api/Dockerfile.alerts b/ee/api/Dockerfile.alerts deleted file mode 100644 index b2e3bf48e..000000000 --- a/ee/api/Dockerfile.alerts +++ /dev/null @@ -1,19 +0,0 @@ -FROM python:3.9.7-slim -LABEL Maintainer="Rajesh Rajendran" -LABEL Maintainer="KRAIEM Taha Yassine" -RUN apt-get update && apt-get install -y pkg-config libxmlsec1-dev gcc && rm -rf /var/lib/apt/lists/* -WORKDIR /work -COPY . . -RUN pip install -r requirements.txt -RUN mv .env.default .env && rm app.py && mv app_alerts.py app.py -ENV pg_minconn 2 - -# Add Tini -# Startup daemon -ENV TINI_VERSION v0.19.0 -ARG envarg -ENV ENTERPRISE_BUILD ${envarg} -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini -RUN chmod +x /tini -ENTRYPOINT ["/tini", "--"] -CMD ./entrypoint.sh From 277d5917fd53395c15cf9214cf0af22e5e51d403 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 13:49:57 +0100 Subject: [PATCH 064/206] feat(api): changed alerts_build script --- api/build_alerts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index 0b877006e..f83090094 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -19,7 +19,7 @@ function make_submodule() { cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part - cp -R ./{Dockerfile.alerts,requirements.txt,.env.default} ./alerts/ + cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default} ./alerts/ } git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} From 56717e4f6b7598bdd33d722d9684e446fa43ea11 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 14:11:44 +0100 Subject: [PATCH 065/206] feat(api): changed alerts_build script --- api/build_alerts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index f83090094..2a8e2efca 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -19,7 +19,7 @@ function make_submodule() { cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part - cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default} ./alerts/ + cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ } git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} From 141cb95b104977fa7a7276a1c7effdff735da5ed Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 14:44:50 +0100 Subject: [PATCH 066/206] feat(api): changed alerts_build script feat(api): changed alerts.yaml feat(api): changed templates/alerts.yaml --- api/build_alerts.sh | 1 + api/requirements_alerts.txt | 1 + scripts/helm/app/alerts.yaml | 24 ++++++++++++++--- .../roles/openreplay/templates/alerts.yaml | 26 ++++++++++++++++++- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index 2a8e2efca..c42c79c10 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -20,6 +20,7 @@ function make_submodule() { # -- end of generated part cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ + cp ./chalicelib/utils/html ./alerts/chalicelib/utils/html } git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} diff --git a/api/requirements_alerts.txt b/api/requirements_alerts.txt index da4f74530..1f749f88d 100644 --- a/api/requirements_alerts.txt +++ b/api/requirements_alerts.txt @@ -1,6 +1,7 @@ requests==2.26.0 urllib3==1.26.6 boto3==1.16.1 +pyjwt==1.7.1 psycopg2-binary==2.8.6 diff --git a/scripts/helm/app/alerts.yaml b/scripts/helm/app/alerts.yaml index f992a7cee..59e6bc18b 100644 --- a/scripts/helm/app/alerts.yaml +++ b/scripts/helm/app/alerts.yaml @@ -22,7 +22,25 @@ resources: memory: 1Mi env: - ALERT_NOTIFICATION_STRING: http://chalice-openreplay.app.svc.cluster.local:8000/alerts/notifications - CLICKHOUSE_STRING: tcp://clickhouse.db.svc.cluster.local:9000/default - POSTGRES_STRING: postgres://postgres:asayerPostgres@postgresql.db.svc.cluster.local:5432 + pg_host: postgresql.db.svc.cluster.local + pg_port: 5432 + pg_dbname: postgres + pg_user: postgres + pg_password: asayerPostgres + EMAIL_HOST: '' + EMAIL_PORT: '587' + EMAIL_USER: '' + EMAIL_PASSWORD: '' + EMAIL_USE_TLS: 'true' + EMAIL_USE_SSL: 'false' + EMAIL_SSL_KEY: '' + EMAIL_SSL_CERT: '' + EMAIL_FROM: OpenReplay + SITE_URL: '' + S3_HOST: 'http://minio.db.svc.cluster.local:9000' + S3_KEY: minios3AccessKeyS3cr3t + S3_SECRET: m1n10s3CretK3yPassw0rd + AWS_DEFAULT_REGION: us-east-1 LICENSE_KEY: "" + PYTHONUNBUFFERED: '0' + version_number: '1.3.6' diff --git a/scripts/helm/roles/openreplay/templates/alerts.yaml b/scripts/helm/roles/openreplay/templates/alerts.yaml index b2a91832b..97178c1ce 100644 --- a/scripts/helm/roles/openreplay/templates/alerts.yaml +++ b/scripts/helm/roles/openreplay/templates/alerts.yaml @@ -4,8 +4,32 @@ image: tag: {{ image_tag }} {% endif %} env: + S3_KEY: "{{ minio_access_key }}" + S3_SECRET: "{{ minio_secret_key }}" + SITE_URL: "https://{{ domain_name }}" + pg_host: "{{ postgres_endpoint }}" + pg_port: "{{ postgres_port }}" + pg_dbname: "{{ postgres_db_name }}" + pg_user: "{{ postgres_db_user }}" + pg_password: "{{ postgres_db_password }}" + EMAIL_HOST: "{{ email_host }}" + EMAIL_PORT: "{{ email_port }}" + EMAIL_USER: "{{ email_user }}" + EMAIL_PASSWORD: "{{ email_password }}" + EMAIL_USE_TLS: "{{ email_use_tls }}" + EMAIL_USE_SSL: "{{ email_use_ssl }}" + EMAIL_SSL_KEY: "{{ email_ssl_key }}" + EMAIL_SSL_CERT: "{{ email_ssl_cert }}" + EMAIL_FROM: "{{ email_from }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" LICENSE_KEY: "{{ enterprise_edition_license }}" - POSTGRES_STRING: "postgres://{{postgres_db_user}}:{{postgres_db_password}}@{{postgres_endpoint}}:{{postgres_port}}/{{ postgres_db_name }}" + # In case of minio, the instance is running inside kuberntes, + # which is accessible via nginx ingress. +{% if s3_endpoint == "http://minio.db.svc.cluster.local:9000" %} + S3_HOST: "https://{{ domain_name }}" +{% else %} + S3_HOST: "{{ s3_endpoint }}" +{% endif %} {% if not (docker_registry_username is defined and docker_registry_username and docker_registry_password is defined and docker_registry_password) %} imagePullSecrets: [] From 7545357dd05d36cb0702161e5d46d1b53e71be3d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 14:55:48 +0100 Subject: [PATCH 067/206] feat(alerts): EE-alerts --- ee/api/.gitignore | 3 + ee/api/build_alerts.sh | 59 +++++ ee/api/chalicelib/core/alerts_processor.py | 265 +++++++++++++++++++++ 3 files changed, 327 insertions(+) create mode 100644 ee/api/build_alerts.sh create mode 100644 ee/api/chalicelib/core/alerts_processor.py diff --git a/ee/api/.gitignore b/ee/api/.gitignore index ee1d296de..6d66f00c5 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -261,3 +261,6 @@ Pipfile /chalicelib/core/custom_metrics.py /chalicelib/core/performance_event.py /chalicelib/core/saved_search.py +/Dockerfile.alerts +/requirements_alerts.txt +/app_alerts.py diff --git a/ee/api/build_alerts.sh b/ee/api/build_alerts.sh new file mode 100644 index 000000000..4a230a358 --- /dev/null +++ b/ee/api/build_alerts.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Script to build alerts module +# flags to accept: +# envarg: build for enterprise edition. +# Default will be OSS build. + +# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh + +function make_submodule() { + # -- this part was generated by modules_lister.py -- + mkdir alerts + cp -R ./{app_alerts}.py ./alerts/ + mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/ + cp -R /Users/tahayk/asayer/openreplay/ee/api/chalicelib/__init__.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/ + mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/core/ + cp -R /Users/tahayk/asayer/openreplay/ee/api/chalicelib/core/{__init__,alerts_processor,sessions,events,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,issues,events_ios,sessions_mobs,errors,dashboard,sourcemaps,sourcemaps_parser,resources,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/core/ + mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/ + cp -R /Users/tahayk/asayer/openreplay/ee/api/schemas.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/ + mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/utils/ + cp -R /Users/tahayk/asayer/openreplay/ee/api/chalicelib/utils/{__init__,TimeUTC,helper,pg_client,event_filter_definition,dev,SAML2_helper,email_helper,email_handler,smtp,s3,args_transformer,ch_client,metrics_helper}.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/utils/ + # -- end of generated part + + cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ + cp ./chalicelib/utils/html ./alerts/chalicelib/utils/html +} + +git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} +envarg="default-foss" +check_prereq() { + which docker || { + echo "Docker not installed, please install docker." + exit=1 + } + [[ exit -eq 1 ]] && exit 1 +} + +function build_api(){ + tag="" + # Copy enterprise code + [[ $1 == "ee" ]] && { + cp -rf ../ee/api/* ./ + envarg="default-ee" + tag="ee-" + } + make_submodule + cd alerts + docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . + cd .. + rm -rf alerts + [[ $PUSH_IMAGE -eq 1 ]] && { + docker push ${DOCKER_REPO:-'local'}/alerts:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/alerts:${git_sha1} ${DOCKER_REPO:-'local'}/alerts:${tag}latest + docker push ${DOCKER_REPO:-'local'}/alerts:${tag}latest + } +} + +check_prereq +build_api $1 \ No newline at end of file diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py new file mode 100644 index 000000000..3bb6b202f --- /dev/null +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -0,0 +1,265 @@ +import schemas +from chalicelib.core import sessions, alerts +from chalicelib.utils import pg_client, helper +from chalicelib.utils.TimeUTC import TimeUTC + +LeftToDb = { + schemas.AlertColumn.performance__dom_content_loaded__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "COALESCE(AVG(NULLIF(dom_content_loaded_time ,0)),0)"}, + schemas.AlertColumn.performance__first_meaningful_paint__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)"}, + schemas.AlertColumn.performance__page_load_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(load_time ,0))"}, + schemas.AlertColumn.performance__dom_build_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(dom_building_time,0))"}, + schemas.AlertColumn.performance__speed_index__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(speed_index,0))"}, + schemas.AlertColumn.performance__page_response_time__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(response_time,0))"}, + schemas.AlertColumn.performance__ttfb__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(first_paint_time,0))"}, + schemas.AlertColumn.performance__time_to_render__average: { + "table": "events.pages INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(visually_complete,0))"}, + schemas.AlertColumn.performance__image_load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='img'"}, + schemas.AlertColumn.performance__request_load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='fetch'"}, + schemas.AlertColumn.resources__load_time__average: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "AVG(NULLIF(resources.duration,0))"}, + schemas.AlertColumn.resources__missing__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE"}, + schemas.AlertColumn.errors__4xx_5xx__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", + "condition": "status/100!=2"}, + schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=4"}, + schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(session_id)", "condition": "status/100=5"}, + schemas.AlertColumn.errors__javascript__impacted_sessions__count: { + "table": "events.resources INNER JOIN public.sessions USING(session_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, + schemas.AlertColumn.performance__crashes__count: { + "table": "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", + "formula": "COUNT(DISTINCT session_id)", "condition": "errors_count > 0"}, + schemas.AlertColumn.errors__javascript__count: { + "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'", "joinSessions": False}, + schemas.AlertColumn.errors__backend__count: { + "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", + "formula": "COUNT(DISTINCT session_id)", "condition": "source!='js_exception'", "joinSessions": False}, +} + +# This is the frequency of execution for each threshold +TimeInterval = { + 15: 3, + 30: 5, + 60: 10, + 120: 20, + 240: 30, + 1440: 60, +} + + +def can_check(a) -> bool: + now = TimeUTC.now() + + repetitionBase = a["options"]["currentPeriod"] \ + if a["detectionMethod"] == schemas.AlertDetectionMethod.change \ + and a["options"]["currentPeriod"] > a["options"]["previousPeriod"] \ + else a["options"]["previousPeriod"] + + if TimeInterval.get(repetitionBase) is None: + print(f"repetitionBase: {repetitionBase} NOT FOUND") + return False + + return (a["options"]["renotifyInterval"] <= 0 or + a["options"].get("lastNotification") is None or + a["options"]["lastNotification"] <= 0 or + ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ + and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 + + +def Build(a): + params = {"project_id": a["projectId"]} + full_args = {} + j_s = True + if a["seriesId"] is not None: + a["filter"]["sort"] = "session_id" + a["filter"]["order"] = "DESC" + a["filter"]["startDate"] = -1 + a["filter"]["endDate"] = TimeUTC.now() + full_args, query_part, sort = sessions.search_query_parts( + data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), + error_status=None, errors_only=False, + favorite_only=False, issue=None, project_id=a["projectId"], + user_id=None) + subQ = f"""SELECT COUNT(session_id) AS value + {query_part}""" + else: + colDef = LeftToDb[a["query"]["left"]] + subQ = f"""SELECT {colDef["formula"]} AS value + FROM {colDef["table"]} + WHERE project_id = %(project_id)s + {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" + j_s = colDef.get("joinSessions", True) + + q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" + + # if len(colDef.group) > 0 { + # subQ = subQ.Column(colDef.group + " AS group_value") + # subQ = subQ.GroupBy(colDef.group) + # q = q.Column("group_value") + # } + + if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: + if a["seriesId"] is not None: + q += f""" FROM ({subQ}) AS stat""" + else: + q += f""" FROM ({subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}) AS stat""" + params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} + else: + if a["options"]["change"] == schemas.AlertDetectionChangeType.change: + # if len(colDef.group) > 0: + # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) + # sub2, args2, _ := subQ.Where( + # sq.And{ + # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), + # }).ToSql() + # sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) + # q = q.FromSelect(sub1, "stat") + # else: + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})-({sub2})) AS value" + q += f" FROM ( {sub1} ) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} + else: + sub1 = f"""{subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND timestamp<%(startDate)s + AND timestamp>=%(timestamp_sub2)s + {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" + params["timestamp_sub2"] = TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000 + sub1 = f"SELECT (( {sub1} )-( {sub2} )) AS value" + q += f" FROM ( {sub1} ) AS stat" + + else: + # if len(colDef.group) >0 { + # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) + # sub2, args2, _ := subQ.Where( + # sq.And{ + # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), + # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), + # }).ToSql() + # sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) + # q = q.FromSelect(sub1, "stat") + # } else { + if a["seriesId"] is not None: + sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") + sub1 = f"SELECT (({subQ})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + params = {**params, **full_args, + "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, + "timestamp_sub2": TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) \ + * 60 * 1000} + else: + sub1 = f"""{subQ} AND timestamp>=%(startDate)s + {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" + params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 + sub2 = f"""{subQ} AND timestamp<%(startDate)s + AND timestamp>=%(timestamp_sub2)s + {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" + params["timestamp_sub2"] = TimeUTC.now() \ + - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) * 60 * 1000 + sub1 = f"SELECT (({sub1})/NULLIF(({sub2}),0)-1)*100 AS value" + q += f" FROM ({sub1}) AS stat" + + return q, params + + +def process(): + notifications = [] + with pg_client.PostgresClient(long_query=True) as cur: + query = """SELECT tenant_id, + alert_id, + project_id, + detection_method, + query, + options, + (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, + alerts.name, + alerts.series_id, + filter + FROM public.alerts + LEFT JOIN metric_series USING (series_id) + INNER JOIN projects USING (project_id) + WHERE alerts.deleted_at ISNULL + AND alerts.active + AND projects.active + AND projects.deleted_at ISNULL + AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) + ORDER BY alerts.created_at;""" + cur.execute(query=query) + all_alerts = helper.list_to_camel_case(cur.fetchall()) + + with pg_client.PostgresClient() as cur: + for alert in all_alerts: + if can_check(alert): + print(f"Querying alertId:{alert['alertId']} name: {alert['name']}") + query, params = Build(alert) + query = cur.mogrify(query, params) + # print(alert) + # print(query) + try: + cur.execute(query) + result = cur.fetchone() + if result["valid"]: + print("Valid alert, notifying users") + notifications.append({ + "alertId": alert["alertId"], + "tenantId": alert["tenantId"], + "title": alert["name"], + "description": f"has been triggered, {alert['query']['left']} = {result['value']} ({alert['query']['operator']} {alert['query']['right']}).", + "buttonText": "Check metrics for more details", + "buttonUrl": f"/{alert['projectId']}/metrics", + "imageUrl": None, + "options": {"source": "ALERT", "sourceId": alert["alertId"], + "sourceMeta": alert["detectionMethod"], + "message": alert["options"]["message"], "projectId": alert["projectId"], + "data": {"title": alert["name"], + "limitValue": alert["query"]["right"], "actualValue": result["value"], + "operator": alert["query"]["operator"], + "trigger": alert["query"]["left"], + "alertId": alert["alertId"], + "detectionMethod": alert["detectionMethod"], + "currentPeriod": alert["options"]["currentPeriod"], + "previousPeriod": alert["options"]["previousPeriod"], + "createdAt": TimeUTC.now()}}, + }) + except Exception as e: + print(f"!!!Error while running alert query for alertId:{alert['alertId']}") + print(str(e)) + print(query) + if len(notifications) > 0: + cur.execute( + cur.mogrify(f"""UPDATE public.Alerts + SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb + WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) + if len(notifications) > 0: + alerts.process_notifications(notifications) From d1de2cb2508432f5e4636428f3337d781e3ce8f4 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 15:15:10 +0100 Subject: [PATCH 068/206] feat(alerts): changed build script --- api/build_alerts.sh | 37 +++++++++++++++++--------- ee/api/build_alerts.sh | 59 ------------------------------------------ 2 files changed, 25 insertions(+), 71 deletions(-) delete mode 100644 ee/api/build_alerts.sh diff --git a/api/build_alerts.sh b/api/build_alerts.sh index c42c79c10..e4ec0dbcc 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -8,17 +8,30 @@ # Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh function make_submodule() { - # -- this part was generated by modules_lister.py -- - mkdir alerts - cp -R ./{app_alerts,schemas}.py ./alerts/ - mkdir -p ./alerts/chalicelib/ - cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ - mkdir -p ./alerts/chalicelib/core/ - cp -R ./chalicelib/core/{__init__,alerts_processor,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,assist,events_ios,sessions_mobs,errors,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ - mkdir -p ./alerts/chalicelib/utils/ - cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ - # -- end of generated part - + [[ $1 != "ee" ]] && { + # -- this part was generated by modules_lister.py -- + mkdir alerts + cp -R ./{app_alerts,schemas}.py ./alerts/ + mkdir -p ./alerts/chalicelib/ + cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ + mkdir -p ./alerts/chalicelib/core/ + cp -R ./chalicelib/core/{__init__,alerts_processor,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,assist,events_ios,sessions_mobs,errors,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ + mkdir -p ./alerts/chalicelib/utils/ + cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ + # -- end of generated part + } + [[ $1 == "ee" ]] && { + # -- this part was generated by modules_lister.py -- + mkdir alerts + cp -R ./{app_alerts,schemas}.py ./alerts/ + mkdir -p ./alerts/chalicelib/ + cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ + mkdir -p ./alerts/chalicelib/core/ + cp -R ./chalicelib/core/{__init__,alerts_processor,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,events_ios,sessions_mobs,errors,dashboard,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ + mkdir -p ./alerts/chalicelib/utils/ + cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,SAML2_helper,email_helper,email_handler,smtp,s3,args_transformer,ch_client,metrics_helper}.py ./alerts/chalicelib/utils/ + # -- end of generated part + } cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ cp ./chalicelib/utils/html ./alerts/chalicelib/utils/html } @@ -41,7 +54,7 @@ function build_api(){ envarg="default-ee" tag="ee-" } - make_submodule + make_submodule $1 cd alerts docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . cd .. diff --git a/ee/api/build_alerts.sh b/ee/api/build_alerts.sh deleted file mode 100644 index 4a230a358..000000000 --- a/ee/api/build_alerts.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Script to build alerts module -# flags to accept: -# envarg: build for enterprise edition. -# Default will be OSS build. - -# Usage: IMAGE_TAG=latest DOCKER_REPO=myDockerHubID bash build.sh - -function make_submodule() { - # -- this part was generated by modules_lister.py -- - mkdir alerts - cp -R ./{app_alerts}.py ./alerts/ - mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/ - cp -R /Users/tahayk/asayer/openreplay/ee/api/chalicelib/__init__.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/ - mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/core/ - cp -R /Users/tahayk/asayer/openreplay/ee/api/chalicelib/core/{__init__,alerts_processor,sessions,events,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,issues,events_ios,sessions_mobs,errors,dashboard,sourcemaps,sourcemaps_parser,resources,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/core/ - mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/ - cp -R /Users/tahayk/asayer/openreplay/ee/api/schemas.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/ - mkdir -p ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/utils/ - cp -R /Users/tahayk/asayer/openreplay/ee/api/chalicelib/utils/{__init__,TimeUTC,helper,pg_client,event_filter_definition,dev,SAML2_helper,email_helper,email_handler,smtp,s3,args_transformer,ch_client,metrics_helper}.py ./alerts/sers/tahayk/asayer/openreplay/ee/api/chalicelib/utils/ - # -- end of generated part - - cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ - cp ./chalicelib/utils/html ./alerts/chalicelib/utils/html -} - -git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} -envarg="default-foss" -check_prereq() { - which docker || { - echo "Docker not installed, please install docker." - exit=1 - } - [[ exit -eq 1 ]] && exit 1 -} - -function build_api(){ - tag="" - # Copy enterprise code - [[ $1 == "ee" ]] && { - cp -rf ../ee/api/* ./ - envarg="default-ee" - tag="ee-" - } - make_submodule - cd alerts - docker build -f ./Dockerfile.alerts --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/alerts:${git_sha1} . - cd .. - rm -rf alerts - [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/alerts:${git_sha1} - docker tag ${DOCKER_REPO:-'local'}/alerts:${git_sha1} ${DOCKER_REPO:-'local'}/alerts:${tag}latest - docker push ${DOCKER_REPO:-'local'}/alerts:${tag}latest - } -} - -check_prereq -build_api $1 \ No newline at end of file From d311330dae9f4f48a3b440ba98f4ba3d361984b0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 15:24:22 +0100 Subject: [PATCH 069/206] feat(alerts): changed build script --- api/build_alerts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index e4ec0dbcc..df6750e20 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -33,7 +33,7 @@ function make_submodule() { # -- end of generated part } cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ - cp ./chalicelib/utils/html ./alerts/chalicelib/utils/html + cp -R ./chalicelib/utils/html ./alerts/chalicelib/utils/html } git_sha1=${IMAGE_TAG:-$(git rev-parse HEAD)} From 30aa7b0d8c9b64633f24d9d292234c0c4a89d443 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 15:34:22 +0100 Subject: [PATCH 070/206] feat(alerts): changed dependencies feat(alerts): changed build script feat(alerts): EE dockerfile --- api/Dockerfile.alerts | 2 +- api/build_alerts.sh | 2 +- api/requirements_alerts.txt | 13 ------------- ee/api/.gitignore | 1 - ee/api/Dockerfile.alerts | 19 +++++++++++++++++++ 5 files changed, 21 insertions(+), 16 deletions(-) delete mode 100644 api/requirements_alerts.txt create mode 100644 ee/api/Dockerfile.alerts diff --git a/api/Dockerfile.alerts b/api/Dockerfile.alerts index ff5e7091e..7bff6a9dc 100644 --- a/api/Dockerfile.alerts +++ b/api/Dockerfile.alerts @@ -3,7 +3,7 @@ LABEL Maintainer="Rajesh Rajendran" LABEL Maintainer="KRAIEM Taha Yassine" WORKDIR /work COPY . . -RUN pip install -r requirements_alerts.txt +RUN pip install -r requirements.txt RUN mv .env.default .env && mv app_alerts.py app.py ENV pg_minconn 2 diff --git a/api/build_alerts.sh b/api/build_alerts.sh index df6750e20..bfe9c22a1 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -32,7 +32,7 @@ function make_submodule() { cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,SAML2_helper,email_helper,email_handler,smtp,s3,args_transformer,ch_client,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part } - cp -R ./{Dockerfile.alerts,requirements_alerts.txt,.env.default,entrypoint.sh} ./alerts/ + cp -R ./{Dockerfile.alerts,requirements.txt,.env.default,entrypoint.sh} ./alerts/ cp -R ./chalicelib/utils/html ./alerts/chalicelib/utils/html } diff --git a/api/requirements_alerts.txt b/api/requirements_alerts.txt deleted file mode 100644 index 1f749f88d..000000000 --- a/api/requirements_alerts.txt +++ /dev/null @@ -1,13 +0,0 @@ -requests==2.26.0 -urllib3==1.26.6 -boto3==1.16.1 -pyjwt==1.7.1 -psycopg2-binary==2.8.6 - - - -fastapi==0.70.1 -uvicorn[standard]==0.16.0 -python-decouple==3.5 -pydantic[email]==1.8.2 -apscheduler==3.8.1 \ No newline at end of file diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 6d66f00c5..f7abe8f5d 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -261,6 +261,5 @@ Pipfile /chalicelib/core/custom_metrics.py /chalicelib/core/performance_event.py /chalicelib/core/saved_search.py -/Dockerfile.alerts /requirements_alerts.txt /app_alerts.py diff --git a/ee/api/Dockerfile.alerts b/ee/api/Dockerfile.alerts new file mode 100644 index 000000000..5809de5e6 --- /dev/null +++ b/ee/api/Dockerfile.alerts @@ -0,0 +1,19 @@ +FROM python:3.9.7-slim +LABEL Maintainer="Rajesh Rajendran" +LABEL Maintainer="KRAIEM Taha Yassine" +RUN apt-get update && apt-get install -y pkg-config libxmlsec1-dev gcc && rm -rf /var/lib/apt/lists/* +WORKDIR /work +COPY . . +RUN pip install -r requirements.txt +RUN mv .env.default .env && mv app_alerts.py app.py +ENV pg_minconn 2 + +# Add Tini +# Startup daemon +ENV TINI_VERSION v0.19.0 +ARG envarg +ENV ENTERPRISE_BUILD ${envarg} +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +CMD ./entrypoint.sh \ No newline at end of file From e844453106262a016096ab99bea8771c31812fdc Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 15:58:21 +0100 Subject: [PATCH 071/206] feat(api): change build script to build alerts --- api/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/build.sh b/api/build.sh index 199448da1..ae4a16e66 100644 --- a/api/build.sh +++ b/api/build.sh @@ -34,4 +34,5 @@ function build_api(){ } check_prereq -build_api $1 \ No newline at end of file +build_api $1 +IMAGE_TAG=$IMAGE_TAG PUSH_IMAGE=$PUSH_IMAGE DOCKER_REPO=$DOCKER_REPO bash build_alerts.sh $1 \ No newline at end of file From c5001638ec4af6dd981986890c9994888e1f7444 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 16:28:10 +0100 Subject: [PATCH 072/206] feat(api): change build script --- api/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/build.sh b/api/build.sh index ae4a16e66..29b8911ca 100644 --- a/api/build.sh +++ b/api/build.sh @@ -25,11 +25,11 @@ function build_api(){ envarg="default-ee" tag="ee-" } - docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/api:${git_sha1} . + docker build -f ./Dockerfile --build-arg envarg=$envarg -t ${DOCKER_REPO:-'local'}/chalice:${git_sha1} . [[ $PUSH_IMAGE -eq 1 ]] && { - docker push ${DOCKER_REPO:-'local'}/api:${git_sha1} - docker tag ${DOCKER_REPO:-'local'}/api:${git_sha1} ${DOCKER_REPO:-'local'}/api:${tag}latest - docker push ${DOCKER_REPO:-'local'}/api:${tag}latest + docker push ${DOCKER_REPO:-'local'}/chalice:${git_sha1} + docker tag ${DOCKER_REPO:-'local'}/chalice:${git_sha1} ${DOCKER_REPO:-'local'}/chalice:${tag}latest + docker push ${DOCKER_REPO:-'local'}/chalice:${tag}latest } } From 1eaf96e097de4f342576eb64cfad458521f02b45 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 17:11:08 +0100 Subject: [PATCH 073/206] feat(api):small change to force EE build --- ee/api/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/api/.gitignore b/ee/api/.gitignore index f7abe8f5d..cfd351eea 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -262,4 +262,4 @@ Pipfile /chalicelib/core/performance_event.py /chalicelib/core/saved_search.py /requirements_alerts.txt -/app_alerts.py +/app_alerts.py \ No newline at end of file From a71d3db54015b1b4809d9c494afee7dbc50666ab Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 10 Jan 2022 18:04:06 +0100 Subject: [PATCH 074/206] feat(DB): db version --- ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 5 +++++ ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 6 ++++++ scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql | 5 +++++ scripts/helm/db/init_dbs/postgresql/init_schema.sql | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 6cb678c96..75130e2f4 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -1,4 +1,9 @@ BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.9.9-ee' +$$ LANGUAGE sql IMMUTABLE; CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 3138e8d19..451c6b2a0 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -3,6 +3,12 @@ BEGIN; CREATE SCHEMA IF NOT EXISTS events_common; CREATE SCHEMA IF NOT EXISTS events; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.9.9-ee' +$$ LANGUAGE sql IMMUTABLE; + -- --- accounts.sql --- CREATE OR REPLACE FUNCTION generate_api_key(length integer) RETURNS text AS diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 3aafca980..712990680 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -1,4 +1,9 @@ BEGIN; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.9.9' +$$ LANGUAGE sql IMMUTABLE; CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 8f74e721e..1432af17c 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -3,6 +3,12 @@ BEGIN; CREATE SCHEMA IF NOT EXISTS events_common; CREATE SCHEMA IF NOT EXISTS events; +CREATE OR REPLACE FUNCTION openreplay_version() + RETURNS text AS +$$ +SELECT 'v1.9.9' +$$ LANGUAGE sql IMMUTABLE; + -- --- accounts.sql --- CREATE OR REPLACE FUNCTION generate_api_key(length integer) RETURNS text AS From f056f690670975eeb5d8e694616cffc422369ff8 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Tue, 11 Jan 2022 09:30:59 +0100 Subject: [PATCH 075/206] feat(tracker-assist): 3.4.13: scroll behaviour improvement & mouse events incapsulation --- tracker/tracker-assist/README.md | 2 +- tracker/tracker-assist/package.json | 2 +- tracker/tracker-assist/src/CallWindow.ts | 8 --- tracker/tracker-assist/src/Mouse.ts | 79 ++++++++++++++++++++++-- tracker/tracker-assist/src/index.ts | 50 ++++----------- 5 files changed, 89 insertions(+), 52 deletions(-) diff --git a/tracker/tracker-assist/README.md b/tracker/tracker-assist/README.md index 2ebac72d4..0c7bfe00f 100644 --- a/tracker/tracker-assist/README.md +++ b/tracker/tracker-assist/README.md @@ -72,7 +72,7 @@ onAgentConnect: () => { Warning: it is possible for the same agent to be connected/disconnected several times during one session due to a bad network. Several agents may connect simultaneously. -A callback `onCallStart` will be fired when the end-user accepts the call. It can return another callback that will be called on call end. +A callback `onCallStart` will be fired when the end-user accepts the call. It can return another callback that will be called on the call end. ```ts onCallStart: () => { console.log("Allo!") diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 97973d6dd..506f05e2d 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.4.11", + "version": "3.4.13", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/CallWindow.ts b/tracker/tracker-assist/src/CallWindow.ts index 3499cf7ea..1299008a8 100644 --- a/tracker/tracker-assist/src/CallWindow.ts +++ b/tracker/tracker-assist/src/CallWindow.ts @@ -207,14 +207,6 @@ export default class CallWindow { private toggleAudio() { const enabled = this.localStream?.toggleAudio() || false this.toggleAudioUI(enabled) - // if (!this.audioBtn) { return; } - // if (enabled) { - // this.audioBtn.classList.remove("muted"); - // this.audioBtn.childNodes[1].textContent = "Mute"; - // } else { - // this.audioBtn.classList.add("muted"); - // this.audioBtn.childNodes[1].textContent = "Unmute"; - // } } private toggleVideoUI(enabled: boolean) { diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index 51fb67e8e..d2c89cfe6 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -1,3 +1,5 @@ +type XY = [number, number] + export default class Mouse { private mouse: HTMLDivElement @@ -14,23 +16,90 @@ export default class Mouse { background: "radial-gradient(red, transparent)", }); document.body.appendChild(this.mouse); + + + window.addEventListener("scroll", this.handleWScroll) + window.addEventListener("resize", this.resetLastScrEl) } - move({x, y}: {x: number, y: number}) { - this.position = [x, y]; + move(pos: XY) { + if (this.position[0] !== pos[0] || this.position[1] !== pos[1]) { + this.resetLastScrEl() + } + + this.position = pos; Object.assign(this.mouse.style, { - left: `${x || 0}px`, - top: `${y || 0}px` + left: `${pos[0] || 0}px`, + top: `${pos[1] || 0}px` }) + } - getPosition(): [ number, number] { + getPosition(): XY { return this.position; } + click(pos: XY) { + const el = document.elementFromPoint(pos[0], pos[1]) + if (el instanceof HTMLElement) { + el.click() + el.focus() + } + } + + private readonly pScrEl = document.scrollingElement || document.documentElement + private lastScrEl: Element | "window" | null = null + private resetLastScrEl = () => { this.lastScrEl = null } + private handleWScroll = e => { + if (e.target !== this.lastScrEl) { + this.resetLastScrEl() + } + } + scroll(delta: XY) { + // what would be the browser-like logic? + const [mouseX, mouseY] = this.position + const [dX, dY] = delta + + let el = this.lastScrEl + // Scroll the same one + if (el instanceof Element) { + el.scrollLeft += dX + el.scrollTop += dY + return // TODO: if not scrolled + } + if (el === "window") { + window.scroll(this.pScrEl.scrollLeft + dX, this.pScrEl.scrollTop + dY) + return + } + + el = document.elementFromPoint( + mouseX-this.pScrEl.scrollLeft, + mouseY-this.pScrEl.scrollTop, + ) + while (el) { + //if(el.scrollWidth > el.clientWidth) // - This check doesn't work in common case + const esl = el.scrollLeft + el.scrollLeft += dX + const est = el.scrollTop + el.scrollTop += dY + if (esl !== el.scrollLeft || est !== el.scrollTop) { + this.lastScrEl = el + return + } else { + el = el.parentElement + } + } + + // If not scrolled + window.scroll(this.pScrEl.scrollLeft + dX, this.pScrEl.scrollTop + dY) + this.lastScrEl = "window" + } + remove() { if (this.mouse.parentElement) { document.body.removeChild(this.mouse); } + window.removeEventListener("scroll", this.handleWScroll) + window.removeEventListener("resize", this.resetLastScrEl) } } \ No newline at end of file diff --git a/tracker/tracker-assist/src/index.ts b/tracker/tracker-assist/src/index.ts index 3ffd4671a..74514378a 100644 --- a/tracker/tracker-assist/src/index.ts +++ b/tracker/tracker-assist/src/index.ts @@ -11,13 +11,13 @@ import ConfirmWindow from './ConfirmWindow.js'; import RequestLocalStream from './LocalStream.js'; export interface Options { - onAgentConnect: () => (()=>{} | void), - onCallStart: () => (()=>{} | void), + onAgentConnect: () => ((()=>{}) | void), + onCallStart: () => ((()=>{}) | void), confirmText: string, confirmStyle: Object, // Styles object session_calling_peer_key: string, config: RTCConfiguration, - __messages_per_send?: number, + // __messages_per_send?: number, } enum CallingState { @@ -27,7 +27,7 @@ enum CallingState { }; //@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules; -Peer = Peer.default || Peer; +//Peer = Peer.default || Peer; // type IncomeMessages = // "call_end" | @@ -102,11 +102,11 @@ export default function(opts?: Partial) { log('Connection opened.') assistDemandedRestart = true; app.stop(); - openDataConnections[conn.peer] = new BufferingConnection(conn, options.__messages_per_send) + openDataConnections[conn.peer] = new BufferingConnection(conn) const onAgentDisconnect = options.onAgentConnect(); conn.on('close', () => { - onAgentDisconnect?.(); + onAgentDisconnect && onAgentDisconnect(); log("Connection close: ", conn.peer) delete openDataConnections[conn.peer] // TODO: check if works properly }) @@ -174,7 +174,7 @@ export default function(opts?: Partial) { let callUI = new CallWindow() const handleCallEnd = () => { - onCallEnd?.() + onCallEnd && onCallEnd() mouse.remove(); callUI.remove(); setCallingState(CallingState.False); @@ -212,46 +212,22 @@ export default function(opts?: Partial) { document.addEventListener("click", onInteraction) }); dataConn.on('data', (data: any) => { + log("Income data: ", data) if (!data) { return } if (data === "call_end") { - log('"call_end" received') - handleCallEnd(); - return; + return handleCallEnd(); } if (data.name === 'string') { - log("Name received: ", data) - callUI.setAssistentName(data.name); + return callUI.setAssistentName(data.name); } if (data.type === "scroll" && Array.isArray(data.delta)) { - const scrEl = document.scrollingElement || document.documentElement - const [mouseX, mouseY] = mouse.getPosition() - const [dX, dY] = data.delta; - const el = document.elementFromPoint(mouseX-scrEl.scrollLeft, mouseY-scrEl.scrollTop) - let scrolled = false // what would be the browser-like logic? - if (el) { - if(el.scrollWidth > el.clientWidth) { - el.scrollLeft += data.delta[0] - scrolled = true - } - if (el && el.scrollHeight > el.clientHeight) { - el.scrollTop += data.delta[1] - scrolled = true - } - } - if (!scrolled) { - window.scroll(scrEl.scrollLeft + data.delta[0], scrEl.scrollTop + data.delta[1]) - } + return mouse.scroll(data.delta) } if (data.type === "click" && typeof data.x === 'number' && typeof data.y === 'number') { - const el = document.elementFromPoint(data.x, data.y) - if (el instanceof HTMLElement) { - el.click() - el.focus() - } - return + return mouse.click([ data.x, data.y ]) } if (typeof data.x === 'number' && typeof data.y === 'number') { - mouse.move(data); + return mouse.move([ data.x, data.y ]) } }); From bb2875b9ee7c88e3ad555e0417a3f570e37ebbcd Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Tue, 11 Jan 2022 09:46:01 +0100 Subject: [PATCH 076/206] frontend(feat): message reading refactored; Assist fixes --- .../MessageDistributor/MessageDistributor.ts | 27 +- .../MessageDistributor/MessageReader.ts | 80 -- .../MessageDistributor/PrimitiveReader.ts | 40 - .../app/player/MessageDistributor/Timed.ts | 5 - .../managers/AssistManager.ts | 181 +---- .../managers/AssistManager_old.ts | 486 +++++++++++ .../MessageDistributor/managers/DOMManager.ts | 16 +- .../MessageDistributor/managers/ListWalker.ts | 2 +- .../managers/MobXStateManager.ts | 14 + .../managers/MouseManager.ts | 6 +- .../managers/PerformanceTrackManager.ts | 10 +- .../managers/StylesManager.ts | 5 +- .../app/player/MessageDistributor/messages.ts | 715 ----------------- .../messages/JSONRawMessageReader.ts | 18 + .../messages/MFileReader.ts | 68 ++ .../messages/MStreamReader.ts | 69 ++ .../messages/PrimitiveReader.ts | 55 ++ .../messages/RawMessageReader.ts | 758 ++++++++++++++++++ .../MessageDistributor/messages/index.ts | 1 + .../MessageDistributor/messages/message.ts | 184 +++++ .../player/MessageDistributor/messages/raw.ts | 491 ++++++++++++ .../MessageDistributor/messages/timed.ts | 1 + .../MessageDistributor/messages/urlResolve.ts | 57 ++ frontend/app/player/ios/Parser.ts | 11 +- 24 files changed, 2274 insertions(+), 1026 deletions(-) delete mode 100644 frontend/app/player/MessageDistributor/MessageReader.ts delete mode 100644 frontend/app/player/MessageDistributor/PrimitiveReader.ts delete mode 100644 frontend/app/player/MessageDistributor/Timed.ts create mode 100644 frontend/app/player/MessageDistributor/managers/AssistManager_old.ts create mode 100644 frontend/app/player/MessageDistributor/managers/MobXStateManager.ts delete mode 100644 frontend/app/player/MessageDistributor/messages.ts create mode 100644 frontend/app/player/MessageDistributor/messages/JSONRawMessageReader.ts create mode 100644 frontend/app/player/MessageDistributor/messages/MFileReader.ts create mode 100644 frontend/app/player/MessageDistributor/messages/MStreamReader.ts create mode 100644 frontend/app/player/MessageDistributor/messages/PrimitiveReader.ts create mode 100644 frontend/app/player/MessageDistributor/messages/RawMessageReader.ts create mode 100644 frontend/app/player/MessageDistributor/messages/index.ts create mode 100644 frontend/app/player/MessageDistributor/messages/message.ts create mode 100644 frontend/app/player/MessageDistributor/messages/raw.ts create mode 100644 frontend/app/player/MessageDistributor/messages/timed.ts create mode 100644 frontend/app/player/MessageDistributor/messages/urlResolve.ts diff --git a/frontend/app/player/MessageDistributor/MessageDistributor.ts b/frontend/app/player/MessageDistributor/MessageDistributor.ts index c742c10b5..8130e635e 100644 --- a/frontend/app/player/MessageDistributor/MessageDistributor.ts +++ b/frontend/app/player/MessageDistributor/MessageDistributor.ts @@ -25,12 +25,11 @@ import WindowNodeCounter from './managers/WindowNodeCounter'; import ActivityManager from './managers/ActivityManager'; import AssistManager from './managers/AssistManager'; -import MessageReader from './MessageReader'; +import MFileReader from './messages/MFileReader'; import { INITIAL_STATE as SUPER_INITIAL_STATE, State as SuperState } from './StatedScreen/StatedScreen'; import { INITIAL_STATE as ASSIST_INITIAL_STATE, State as AssistState } from './managers/AssistManager'; -import type { TimedMessage } from './Timed'; import type { PerformanceChartPoint } from './managers/PerformanceTrackManager'; import type { SkipInterval } from './managers/ActivityManager'; @@ -82,32 +81,24 @@ import type { SetViewportScroll, } from './messages'; -interface Timed { //TODO: to common space - time: number; -} - -type ReduxDecoded = Timed & { - action: {}, - state: {}, - duration: number, -} +import type { Timed } from './messages/timed'; export default class MessageDistributor extends StatedScreen { // TODO: consistent with the other data-lists private readonly locationEventManager: ListWalker/**/ = new ListWalker(); - private readonly locationManager: ListWalker = new ListWalker(); - private readonly loadedLocationManager: ListWalker = new ListWalker(); - private readonly connectionInfoManger: ListWalker = new ListWalker(); + private readonly locationManager: ListWalker = new ListWalker(); + private readonly loadedLocationManager: ListWalker = new ListWalker(); + private readonly connectionInfoManger: ListWalker = new ListWalker(); private readonly performanceTrackManager: PerformanceTrackManager = new PerformanceTrackManager(); private readonly windowNodeCounter: WindowNodeCounter = new WindowNodeCounter(); private readonly clickManager: ListWalker = new ListWalker(); - private readonly resizeManager: ListWalker = new ListWalker([]); + private readonly resizeManager: ListWalker = new ListWalker([]); private readonly pagesManager: PagesManager; private readonly mouseManager: MouseManager; private readonly assistManager: AssistManager; - private readonly scrollManager: ListWalker = new ListWalker(); + private readonly scrollManager: ListWalker = new ListWalker(); private readonly decoder = new Decoder(); private readonly lists = initLists(); @@ -184,7 +175,7 @@ export default class MessageDistributor extends StatedScreen { window.fetch(fileUrl) .then(r => r.arrayBuffer()) .then(b => { - const r = new MessageReader(new Uint8Array(b), this.sessionStart); + const r = new MFileReader(new Uint8Array(b), this.sessionStart); const msgs: Array = []; while (r.hasNext()) { @@ -334,7 +325,7 @@ export default class MessageDistributor extends StatedScreen { } /* Binded */ - distributeMessage = (msg: TimedMessage, index: number): void => { + distributeMessage = (msg: Message, index: number): void => { if ([ "mouse_move", "mouse_click", diff --git a/frontend/app/player/MessageDistributor/MessageReader.ts b/frontend/app/player/MessageDistributor/MessageReader.ts deleted file mode 100644 index dea8759c9..000000000 --- a/frontend/app/player/MessageDistributor/MessageReader.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { TimedMessage, Indexed } from './Timed'; - -import logger from 'App/logger'; -import readMessage, { Message } from './messages'; -import PrimitiveReader from './PrimitiveReader'; - -// function needSkipMessage(data: Uint8Array, p: number, pLast: number): boolean { -// for (let i = 7; i >= 0; i--) { -// if (data[ p + i ] !== data[ pLast + i ]) { -// return data[ p + i ] - data[ pLast + i ] < 0 -// } -// } -// return true -// } - -export default class MessageReader extends PrimitiveReader { - private pLastMessageID: number = 0; - private currentTime: number = 0; - public error: boolean = false; - constructor(data: Uint8Array, private readonly startTime: number) { - super(data); - } - - private needSkipMessage(): boolean { - if (this.p === 0) return false; - for (let i = 7; i >= 0; i--) { - if (this.buf[ this.p + i ] !== this.buf[ this.pLastMessageID + i ]) { - return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0; - } - } - return true; - } - - private readMessage(): Message | null { - this.skip(8); - try { - let msg - msg = readMessage(this); - return msg; - } catch (e) { - this.error = true; - logger.error("Read message error:", e); - return null; - } - } - - hasNext():boolean { - return !this.error && this.buf.length > this.p; - } - - next(): [ TimedMessage, number] | null { - if (!this.hasNext()) { - return null; - } - - while (this.needSkipMessage()) { - this.readMessage(); - } - this.pLastMessageID = this.p; - - const msg = this.readMessage(); - if (!msg) { - return null; - } - - if (msg.tp === "timestamp") { - // if (this.startTime == null) { - // this.startTime = msg.timestamp - // } - this.currentTime = msg.timestamp - this.startTime; - } else { - const tMsg = Object.assign(msg, { - time: this.currentTime, - _index: this.pLastMessageID, - }) - return [tMsg, this.pLastMessageID]; - } - return null; - } -} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/PrimitiveReader.ts b/frontend/app/player/MessageDistributor/PrimitiveReader.ts deleted file mode 100644 index b49955074..000000000 --- a/frontend/app/player/MessageDistributor/PrimitiveReader.ts +++ /dev/null @@ -1,40 +0,0 @@ -export default class PrimitiveReader { - protected p = 0 - constructor(protected readonly buf: Uint8Array) {} - - hasNext() { - return this.p < this.buf.length - } - - readUint() { - var r = 0, s = 1, b; - do { - b = this.buf[this.p++]; - r += (b & 0x7F) * s; - s *= 128; - } while (b >= 0x80) - return r; - } - - readInt() { - let u = this.readUint(); - if (u % 2) { - u = (u + 1) / -2; - } else { - u = u / 2; - } - return u; - } - - readString() { - var l = this.readUint(); - return new TextDecoder().decode(this.buf.subarray(this.p, this.p+=l)); - } - - readBoolean() { - return !!this.buf[this.p++]; - } - skip(n: number) { - this.p += n; - } -} diff --git a/frontend/app/player/MessageDistributor/Timed.ts b/frontend/app/player/MessageDistributor/Timed.ts deleted file mode 100644 index e0a1d6a82..000000000 --- a/frontend/app/player/MessageDistributor/Timed.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { Message } from './messages'; - -export interface Timed { readonly time: number }; -export interface Indexed { readonly _index: number }; // TODO: remove dash (evwrywhere) -export type TimedMessage = Timed & Message; diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager.ts b/frontend/app/player/MessageDistributor/managers/AssistManager.ts index 2ccea1ad5..d4bb40b0a 100644 --- a/frontend/app/player/MessageDistributor/managers/AssistManager.ts +++ b/frontend/app/player/MessageDistributor/managers/AssistManager.ts @@ -1,14 +1,14 @@ import type Peer from 'peerjs'; import type { DataConnection, MediaConnection } from 'peerjs'; import type MessageDistributor from '../MessageDistributor'; -import type { TimedMessage } from '../Timed'; import type { Message } from '../messages' -import { ID_TP_MAP } from '../messages'; import store from 'App/store'; import type { LocalStream } from './LocalStream'; import { update, getState } from '../../store'; import { iceServerConfigFromString } from 'App/utils' +import MStreamReader from '../messages/MStreamReader';; +import JSONRawMessageReader from '../messages/JSONRawMessageReader' export enum CallingState { Reconnecting, @@ -59,68 +59,15 @@ export const INITIAL_STATE: State = { const MAX_RECONNECTION_COUNT = 4; -function resolveURL(baseURL: string, relURL: string): string { - if (relURL.startsWith('#') || relURL === "") { - return relURL; - } - return new URL(relURL, baseURL).toString(); -} - - -var match = /bar/.exec("foobar"); -const re1 = /url\(("[^"]*"|'[^']*'|[^)]*)\)/g -const re2 = /@import "(.*?)"/g -function cssUrlsIndex(css: string): Array<[number, number]> { - const idxs: Array<[number, number]> = []; - const i1 = css.matchAll(re1); - // @ts-ignore - for (let m of i1) { - // @ts-ignore - const s: number = m.index + m[0].indexOf(m[1]); - const e: number = s + m[1].length; - idxs.push([s, e]); - } - const i2 = css.matchAll(re2); - // @ts-ignore - for (let m of i2) { - // @ts-ignore - const s = m.index + m[0].indexOf(m[1]); - const e = s + m[1].length; - idxs.push([s, e]) - } - return idxs; -} -function unquote(str: string): [string, string] { - str = str.trim(); - if (str.length <= 2) { - return [str, ""] - } - if (str[0] == '"' && str[str.length-1] == '"') { - return [ str.substring(1, str.length-1), "\""]; - } - if (str[0] == '\'' && str[str.length-1] == '\'') { - return [ str.substring(1, str.length-1), "'" ]; - } - return [str, ""] -} -function rewriteCSSLinks(css: string, rewriter: (rawurl: string) => string): string { - for (let idx of cssUrlsIndex(css)) { - const f = idx[0] - const t = idx[1] - const [ rawurl, q ] = unquote(css.substring(f, t)); - css = css.substring(0,f) + q + rewriter(rawurl) + q + css.substring(t); - } - return css -} - -function resolveCSS(baseURL: string, css: string): string { - return rewriteCSSLinks(css, rawurl => resolveURL(baseURL, rawurl)); -} - export default class AssistManager { constructor(private session, private md: MessageDistributor, private config) {} private setStatus(status: ConnectionStatus) { + if (getState().peerConnectionStatus === ConnectionStatus.Disconnected && + status !== ConnectionStatus.Connected) { + return + } + if (status === ConnectionStatus.Connecting) { this.md.setMessagesLoading(true); } else { @@ -148,6 +95,7 @@ export default class AssistManager { } this.setStatus(ConnectionStatus.Connecting) import('peerjs').then(({ default: Peer }) => { + if (this.closed) {return} const _config = { // @ts-ignore host: new URL(window.ENV.API_EDP).host, @@ -170,12 +118,11 @@ export default class AssistManager { console.warn("AssistManager PeerJS peer error: ", e.type, e) } if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { - if (this.peer && this.connectionAttempts++ < MAX_RECONNECTION_COUNT) { - this.setStatus(ConnectionStatus.Connecting); + if (this.peer) { + this.setStatus(this.connectionAttempts++ < MAX_RECONNECTION_COUNT + ? ConnectionStatus.Connecting + : ConnectionStatus.Disconnected); this.connectToPeer(); - } else { - this.setStatus(ConnectionStatus.Disconnected); - this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID); } } else { console.error(`PeerJS error (on peer). Type ${e.type}`, e); @@ -190,12 +137,11 @@ export default class AssistManager { }); } - private dataCheckIntervalID: ReturnType | undefined; private connectToPeer() { if (!this.peer) { return; } this.setStatus(ConnectionStatus.Connecting); const id = this.peerID; - const conn = this.peer.connect(id, { serialization: 'json', reliable: true}); + const conn = this.peer.connect(id, { serialization: "json", reliable: true}); conn.on('open', () => { window.addEventListener("beforeunload", ()=>conn.open &&conn.send("unload")); @@ -206,75 +152,42 @@ export default class AssistManager { this._call() } - let i = 0; let firstMessage = true; this.setStatus(ConnectionStatus.WaitingMessages) + const jmr = new JSONRawMessageReader() + const reader = new MStreamReader(jmr) + conn.on('data', (data) => { - if (!Array.isArray(data)) { return this.handleCommand(data); } this.disconnectTimeout && clearTimeout(this.disconnectTimeout); + + + if (Array.isArray(data)) { + jmr.append(data) // as RawMessage[] + } else if (data instanceof ArrayBuffer) { + //rawMessageReader.append(new Uint8Array(data)) + } else { return this.handleCommand(data); } + if (firstMessage) { firstMessage = false; this.setStatus(ConnectionStatus.Connected) } - let time = 0; - let ts0 = 0; - (data as Array).forEach(msg => { - - // TODO: more appropriate way to do it. - if (msg._id === 60) { - // @ts-ignore - if (msg.name === 'src' || msg.name === 'href') { - // @ts-ignore - msg.value = resolveURL(msg.baseURL, msg.value); - // @ts-ignore - } else if (msg.name === 'style') { - // @ts-ignore - msg.value = resolveCSS(msg.baseURL, msg.value); - } - msg._id = 12; - } else if (msg._id === 61) { // "SetCSSDataURLBased" - // @ts-ignore - msg.data = resolveCSS(msg.baseURL, msg.data); - msg._id = 15; - } else if (msg._id === 67) { // "insert_rule" - // @ts-ignore - msg.rule = resolveCSS(msg.baseURL, msg.rule); - msg._id = 37; - } - - - msg.tp = ID_TP_MAP[msg._id]; // _id goes from tracker - - if (msg.tp === "timestamp") { - ts0 = ts0 || msg.timestamp - time = msg.timestamp - ts0; - return; - } - const tMsg: TimedMessage = Object.assign(msg, { - time, - _index: i, - }); - this.md.distributeMessage(tMsg, i++); - }); + for (let msg = reader.readNext();msg !== null;msg = reader.readNext()) { + //@ts-ignore + this.md.distributeMessage(msg, msg._index); + } }); }); const onDataClose = () => { this.onCallDisconnect() - //console.log('closed peer conn. Reconnecting...') this.connectToPeer(); } - // this.dataCheckIntervalID = setInterval(() => { - // if (!this.dataConnection && getState().peerConnectionStatus === ConnectionStatus.Connected) { - // onDataClose(); - // } - // }, 3000); - conn.on('close', onDataClose);// Does it work ? + conn.on('close', onDataClose);// What case does it work ? conn.on("error", (e) => { this.setStatus(ConnectionStatus.Error); }) @@ -284,11 +197,9 @@ export default class AssistManager { private get dataConnection(): DataConnection | undefined { return this.peer?.connections[this.peerID]?.find(c => c.type === 'data' && c.open); } - private get callConnection(): MediaConnection | undefined { return this.peer?.connections[this.peerID]?.find(c => c.type === 'media' && c.open); } - private send(data: any) { this.dataConnection?.send(data); } @@ -326,18 +237,20 @@ export default class AssistManager { private disconnectTimeout: ReturnType | undefined; + private closeDataConnectionTimeout: ReturnType | undefined; private handleCommand(command: string) { console.log("Data command", command) switch (command) { case "unload": //this.onTrackerCallEnd(); - this.onCallDisconnect() - this.dataConnection?.close(); + this.closeDataConnectionTimeout = setTimeout(() => { + this.onCallDisconnect() + this.dataConnection?.close(); + }, 1500); this.disconnectTimeout = setTimeout(() => { this.onTrackerCallEnd(); this.setStatus(ConnectionStatus.Disconnected); }, 15000); // TODO: more convenient way - //this.dataConnection?.close(); return; case "call_end": this.onTrackerCallEnd(); @@ -349,29 +262,17 @@ export default class AssistManager { } } - // private mmtid?:ReturnType private onMouseMove = (e: MouseEvent): void => { - // this.mmtid && clearTimeout(this.mmtid) - // this.mmtid = setTimeout(() => { const data = this.md.getInternalCoordinates(e); this.send({ x: Math.round(data.x), y: Math.round(data.y) }); - // }, 5) } - // private wtid?: ReturnType - // private scrollDelta: [number, number] = [0,0] private onWheel = (e: WheelEvent): void => { e.preventDefault() - //throttling makes movements less smooth - // this.wtid && clearTimeout(this.wtid) - // this.scrollDelta[0] += e.deltaX - // this.scrollDelta[1] += e.deltaY - // this.wtid = setTimeout(() => { - this.send({ type: "scroll", delta: [ e.deltaX, e.deltaY ]})//this.scrollDelta }); - this.onMouseMove(e) - // this.scrollDelta = [0,0] - // }, 20) + //throttling makes movements less smooth, so it is omitted + //this.onMouseMove(e) + this.send({ type: "scroll", delta: [ e.deltaX, e.deltaY ]}) } private onMouseClick = (e: MouseEvent): void => { @@ -459,7 +360,7 @@ export default class AssistManager { }); this.md.overlay.addEventListener("mousemove", this.onMouseMove) - // this.md.overlay.addEventListener("click", this.onMouseClick) + this.md.overlay.addEventListener("click", this.onMouseClick) }); //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) @@ -473,13 +374,15 @@ export default class AssistManager { window.addEventListener("beforeunload", this.initiateCallEnd) } + closed = false clear() { + this.closed =true this.initiateCallEnd(); - this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID); if (this.peer) { - //console.log("destroying peer...") + console.log("destroying peer...") const peer = this.peer; // otherwise it calls reconnection on data chan close this.peer = null; + peer.disconnect(); peer.destroy(); } } diff --git a/frontend/app/player/MessageDistributor/managers/AssistManager_old.ts b/frontend/app/player/MessageDistributor/managers/AssistManager_old.ts new file mode 100644 index 000000000..b901dc076 --- /dev/null +++ b/frontend/app/player/MessageDistributor/managers/AssistManager_old.ts @@ -0,0 +1,486 @@ +// import type Peer from 'peerjs'; +// import type { DataConnection, MediaConnection } from 'peerjs'; +// import type MessageDistributor from '../MessageDistributor'; +// import type { Message } from '../messages' +// import store from 'App/store'; +// import type { LocalStream } from './LocalStream'; +// import { update, getState } from '../../store'; +// import { iceServerConfigFromString } from 'App/utils' + + +// export enum CallingState { +// Reconnecting, +// Requesting, +// True, +// False, +// }; + +// export enum ConnectionStatus { +// Connecting, +// WaitingMessages, +// Connected, +// Inactive, +// Disconnected, +// Error, +// }; + + +// export function getStatusText(status: ConnectionStatus): string { +// switch(status) { +// case ConnectionStatus.Connecting: +// return "Connecting..."; +// case ConnectionStatus.Connected: +// return ""; +// case ConnectionStatus.Inactive: +// return "Client tab is inactive"; +// case ConnectionStatus.Disconnected: +// return "Disconnected"; +// case ConnectionStatus.Error: +// return "Something went wrong. Try to reload the page."; +// case ConnectionStatus.WaitingMessages: +// return "Connected. Waiting for the data... (The tab might be inactive)" +// } +// } + +// export interface State { +// calling: CallingState, +// peerConnectionStatus: ConnectionStatus, +// remoteControl: boolean, +// } + +// export const INITIAL_STATE: State = { +// calling: CallingState.False, +// peerConnectionStatus: ConnectionStatus.Connecting, +// remoteControl: false, +// } + +// const MAX_RECONNECTION_COUNT = 4; + + +// function resolveURL(baseURL: string, relURL: string): string { +// if (relURL.startsWith('#') || relURL === "") { +// return relURL; +// } +// return new URL(relURL, baseURL).toString(); +// } + + +// var match = /bar/.exec("foobar"); +// const re1 = /url\(("[^"]*"|'[^']*'|[^)]*)\)/g +// const re2 = /@import "(.*?)"/g +// function cssUrlsIndex(css: string): Array<[number, number]> { +// const idxs: Array<[number, number]> = []; +// const i1 = css.matchAll(re1); +// // @ts-ignore +// for (let m of i1) { +// // @ts-ignore +// const s: number = m.index + m[0].indexOf(m[1]); +// const e: number = s + m[1].length; +// idxs.push([s, e]); +// } +// const i2 = css.matchAll(re2); +// // @ts-ignore +// for (let m of i2) { +// // @ts-ignore +// const s = m.index + m[0].indexOf(m[1]); +// const e = s + m[1].length; +// idxs.push([s, e]) +// } +// return idxs; +// } +// function unquote(str: string): [string, string] { +// str = str.trim(); +// if (str.length <= 2) { +// return [str, ""] +// } +// if (str[0] == '"' && str[str.length-1] == '"') { +// return [ str.substring(1, str.length-1), "\""]; +// } +// if (str[0] == '\'' && str[str.length-1] == '\'') { +// return [ str.substring(1, str.length-1), "'" ]; +// } +// return [str, ""] +// } +// function rewriteCSSLinks(css: string, rewriter: (rawurl: string) => string): string { +// for (let idx of cssUrlsIndex(css)) { +// const f = idx[0] +// const t = idx[1] +// const [ rawurl, q ] = unquote(css.substring(f, t)); +// css = css.substring(0,f) + q + rewriter(rawurl) + q + css.substring(t); +// } +// return css +// } + +// function resolveCSS(baseURL: string, css: string): string { +// return rewriteCSSLinks(css, rawurl => resolveURL(baseURL, rawurl)); +// } + +// export default class AssistManager { +// constructor(private session, private md: MessageDistributor, private config) {} + +// private setStatus(status: ConnectionStatus) { +// if (status === ConnectionStatus.Connecting) { +// this.md.setMessagesLoading(true); +// } else { +// this.md.setMessagesLoading(false); +// } +// if (status === ConnectionStatus.Connected) { +// this.md.display(true); +// } else { +// this.md.display(false); +// } +// update({ peerConnectionStatus: status }); +// } + +// private get peerID(): string { +// return `${this.session.projectKey}-${this.session.sessionId}` +// } + +// private peer: Peer | null = null; +// connectionAttempts: number = 0; +// private peeropened: boolean = false; +// connect() { +// if (this.peer != null) { +// console.error("AssistManager: trying to connect more than once"); +// return; +// } +// this.setStatus(ConnectionStatus.Connecting) +// import('peerjs').then(({ default: Peer }) => { +// const _config = { +// // @ts-ignore +// host: new URL(window.ENV.API_EDP).host, +// path: '/assist', +// port: location.protocol === 'https:' ? 443 : 80, +// } + +// if (this.config) { +// _config['config'] = { +// iceServers: this.config, +// sdpSemantics: 'unified-plan', +// iceTransportPolicy: 'relay', +// }; +// } + +// const peer = new Peer(_config); +// this.peer = peer; +// peer.on('error', e => { +// if (e.type !== 'peer-unavailable') { +// console.warn("AssistManager PeerJS peer error: ", e.type, e) +// } +// if (['peer-unavailable', 'network', 'webrtc'].includes(e.type)) { +// if (this.peer && this.connectionAttempts++ < MAX_RECONNECTION_COUNT) { +// this.setStatus(ConnectionStatus.Connecting); +// this.connectToPeer(); +// } else { +// this.setStatus(ConnectionStatus.Disconnected); +// this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID); +// } +// } else { +// console.error(`PeerJS error (on peer). Type ${e.type}`, e); +// this.setStatus(ConnectionStatus.Error) +// } +// }) +// peer.on("open", () => { +// if (this.peeropened) { return; } +// this.peeropened = true; +// this.connectToPeer(); +// }); +// }); +// } + +// private dataCheckIntervalID: ReturnType | undefined; +// private connectToPeer() { +// if (!this.peer) { return; } +// this.setStatus(ConnectionStatus.Connecting); +// const id = this.peerID; +// const conn = this.peer.connect(id, { serialization: 'json', reliable: true}); +// conn.on('open', () => { +// window.addEventListener("beforeunload", ()=>conn.open &&conn.send("unload")); + +// //console.log("peer connected") + + +// if (getState().calling === CallingState.Reconnecting) { +// this._call() +// } + +// let i = 0; +// let firstMessage = true; + +// this.setStatus(ConnectionStatus.WaitingMessages) + +// conn.on('data', (data) => { +// if (!Array.isArray(data)) { return this.handleCommand(data); } +// this.disconnectTimeout && clearTimeout(this.disconnectTimeout); +// if (firstMessage) { +// firstMessage = false; +// this.setStatus(ConnectionStatus.Connected) +// } + +// let time = 0; +// let ts0 = 0; +// (data as Array).forEach(msg => { + +// // TODO: more appropriate way to do it. +// if (msg._id === 60) { +// // @ts-ignore +// if (msg.name === 'src' || msg.name === 'href') { +// // @ts-ignore +// msg.value = resolveURL(msg.baseURL, msg.value); +// // @ts-ignore +// } else if (msg.name === 'style') { +// // @ts-ignore +// msg.value = resolveCSS(msg.baseURL, msg.value); +// } +// msg._id = 12; +// } else if (msg._id === 61) { // "SetCSSDataURLBased" +// // @ts-ignore +// msg.data = resolveCSS(msg.baseURL, msg.data); +// msg._id = 15; +// } else if (msg._id === 67) { // "insert_rule" +// // @ts-ignore +// msg.rule = resolveCSS(msg.baseURL, msg.rule); +// msg._id = 37; +// } + + +// msg.tp = ID_TP_MAP[msg._id]; // _id goes from tracker + +// if (msg.tp === "timestamp") { +// ts0 = ts0 || msg.timestamp +// time = msg.timestamp - ts0; +// return; +// } +// const tMsg: TimedMessage = Object.assign(msg, { +// time, +// _index: i, +// }); +// this.md.distributeMessage(tMsg, i++); +// }); +// }); +// }); + + +// const onDataClose = () => { +// this.onCallDisconnect() +// //console.log('closed peer conn. Reconnecting...') +// this.connectToPeer(); +// } + +// // this.dataCheckIntervalID = setInterval(() => { +// // if (!this.dataConnection && getState().peerConnectionStatus === ConnectionStatus.Connected) { +// // onDataClose(); +// // } +// // }, 3000); +// conn.on('close', onDataClose);// Does it work ? +// conn.on("error", (e) => { +// this.setStatus(ConnectionStatus.Error); +// }) +// } + + +// private get dataConnection(): DataConnection | undefined { +// return this.peer?.connections[this.peerID]?.find(c => c.type === 'data' && c.open); +// } + +// private get callConnection(): MediaConnection | undefined { +// return this.peer?.connections[this.peerID]?.find(c => c.type === 'media' && c.open); +// } + +// private send(data: any) { +// this.dataConnection?.send(data); +// } + + +// private forceCallEnd() { +// this.callConnection?.close(); +// } +// private notifyCallEnd() { +// const dataConn = this.dataConnection; +// if (dataConn) { +// dataConn.send("call_end"); +// } +// } +// private initiateCallEnd = () => { +// this.forceCallEnd(); +// this.notifyCallEnd(); +// this.localCallData && this.localCallData.onCallEnd(); +// } + +// private onTrackerCallEnd = () => { +// console.log('onTrackerCallEnd') +// this.forceCallEnd(); +// if (getState().calling === CallingState.Requesting) { +// this.localCallData && this.localCallData.onReject(); +// } +// this.localCallData && this.localCallData.onCallEnd(); +// } + +// private onCallDisconnect = () => { +// if (getState().calling === CallingState.True) { +// update({ calling: CallingState.Reconnecting }); +// } +// } + + +// private disconnectTimeout: ReturnType | undefined; +// private handleCommand(command: string) { +// console.log("Data command", command) +// switch (command) { +// case "unload": +// //this.onTrackerCallEnd(); +// this.onCallDisconnect() +// this.dataConnection?.close(); +// this.disconnectTimeout = setTimeout(() => { +// this.onTrackerCallEnd(); +// this.setStatus(ConnectionStatus.Disconnected); +// }, 15000); // TODO: more convenient way +// //this.dataConnection?.close(); +// return; +// case "call_end": +// this.onTrackerCallEnd(); +// return; +// case "call_error": +// this.onTrackerCallEnd(); +// this.setStatus(ConnectionStatus.Error); +// return; +// } +// } + +// // private mmtid?:ReturnType +// private onMouseMove = (e: MouseEvent): void => { +// // this.mmtid && clearTimeout(this.mmtid) +// // this.mmtid = setTimeout(() => { +// const data = this.md.getInternalCoordinates(e); +// this.send({ x: Math.round(data.x), y: Math.round(data.y) }); +// // }, 5) +// } + + +// // private wtid?: ReturnType +// // private scrollDelta: [number, number] = [0,0] +// private onWheel = (e: WheelEvent): void => { +// e.preventDefault() +// //throttling makes movements less smooth +// // this.wtid && clearTimeout(this.wtid) +// // this.scrollDelta[0] += e.deltaX +// // this.scrollDelta[1] += e.deltaY +// // this.wtid = setTimeout(() => { +// this.send({ type: "scroll", delta: [ e.deltaX, e.deltaY ]})//this.scrollDelta }); +// this.onMouseMove(e) +// // this.scrollDelta = [0,0] +// // }, 20) +// } + +// private onMouseClick = (e: MouseEvent): void => { +// const conn = this.dataConnection; +// if (!conn) { return; } +// const data = this.md.getInternalCoordinates(e); +// // const el = this.md.getElementFromPoint(e); // requires requestiong node_id from domManager +// const el = this.md.getElementFromInternalPoint(data) +// if (el instanceof HTMLElement) { +// el.focus() +// el.oninput = e => e.preventDefault(); +// el.onkeydown = e => e.preventDefault(); +// } +// conn.send({ type: "click", x: Math.round(data.x), y: Math.round(data.y) }); +// } + +// private toggleRemoteControl = (flag?: boolean) => { +// const state = getState().remoteControl; +// const newState = typeof flag === 'boolean' ? flag : !state; +// if (state === newState) { return } +// if (newState) { +// this.md.overlay.addEventListener("click", this.onMouseClick); +// this.md.overlay.addEventListener("wheel", this.onWheel) +// update({ remoteControl: true }) +// } else { +// this.md.overlay.removeEventListener("click", this.onMouseClick); +// this.md.overlay.removeEventListener("wheel", this.onWheel); +// update({ remoteControl: false }) +// } +// } + +// private localCallData: { +// localStream: LocalStream, +// onStream: (s: MediaStream)=>void, +// onCallEnd: () => void, +// onReject: () => void, +// onError?: ()=> void +// } | null = null + +// call(localStream: LocalStream, onStream: (s: MediaStream)=>void, onCallEnd: () => void, onReject: () => void, onError?: ()=> void): { end: Function, toggleRemoteControl: Function } { +// this.localCallData = { +// localStream, +// onStream, +// onCallEnd: () => { +// onCallEnd(); +// this.toggleRemoteControl(false); +// this.md.overlay.removeEventListener("mousemove", this.onMouseMove); +// this.md.overlay.removeEventListener("click", this.onMouseClick); +// update({ calling: CallingState.False }); +// this.localCallData = null; +// }, +// onReject, +// onError, +// } +// this._call() +// return { +// end: this.initiateCallEnd, +// toggleRemoteControl: this.toggleRemoteControl, +// } +// } + +// private _call() { +// if (!this.peer || !this.localCallData || ![CallingState.False, CallingState.Reconnecting].includes(getState().calling)) { return null; } + +// update({ calling: CallingState.Requesting }); + +// //console.log('calling...', this.localCallData.localStream) + +// const call = this.peer.call(this.peerID, this.localCallData.localStream.stream); +// this.localCallData.localStream.onVideoTrack(vTrack => { +// const sender = call.peerConnection.getSenders().find(s => s.track?.kind === "video") +// if (!sender) { +// //logger.warn("No video sender found") +// return +// } +// //logger.log("sender found:", sender) +// sender.replaceTrack(vTrack) +// }) + +// call.on('stream', stream => { +// update({ calling: CallingState.True }); +// this.localCallData && this.localCallData.onStream(stream); +// this.send({ +// name: store.getState().getIn([ 'user', 'account', 'name']), +// }); + +// this.md.overlay.addEventListener("mousemove", this.onMouseMove) +// // this.md.overlay.addEventListener("click", this.onMouseClick) +// }); +// //call.peerConnection.addEventListener("track", e => console.log('newtrack',e.track)) + +// call.on("close", this.localCallData.onCallEnd); +// call.on("error", (e) => { +// console.error("PeerJS error (on call):", e) +// this.initiateCallEnd(); +// this.localCallData && this.localCallData.onError && this.localCallData.onError(); +// }); + +// window.addEventListener("beforeunload", this.initiateCallEnd) +// } + +// clear() { +// this.initiateCallEnd(); +// this.dataCheckIntervalID && clearInterval(this.dataCheckIntervalID); +// if (this.peer) { +// //console.log("destroying peer...") +// const peer = this.peer; // otherwise it calls reconnection on data chan close +// this.peer = null; +// peer.destroy(); +// } +// } +// } + + diff --git a/frontend/app/player/MessageDistributor/managers/DOMManager.ts b/frontend/app/player/MessageDistributor/managers/DOMManager.ts index f226c1b4e..7c40a4668 100644 --- a/frontend/app/player/MessageDistributor/managers/DOMManager.ts +++ b/frontend/app/player/MessageDistributor/managers/DOMManager.ts @@ -1,24 +1,22 @@ import type StatedScreen from '../StatedScreen'; import type { Message, SetNodeScroll, CreateElementNode } from '../messages'; -import type { TimedMessage } from '../Timed'; import logger from 'App/logger'; import StylesManager, { rewriteNodeStyleSheet } from './StylesManager'; import ListWalker from './ListWalker'; -import type { Timed }from '../Timed'; const IGNORED_ATTRS = [ "autocomplete", "name" ]; const ATTR_NAME_REGEXP = /([^\t\n\f \/>"'=]+)/; // regexp costs ~ -export default class DOMManager extends ListWalker { +export default class DOMManager extends ListWalker { private isMobile: boolean; private screen: StatedScreen; private nl: Array = []; private isLink: Array = []; // Optimisations private bodyId: number = -1; private postponedBodyMessage: CreateElementNode | null = null; - private nodeScrollManagers: Array> = []; + private nodeScrollManagers: Array> = []; private stylesManager: StylesManager; @@ -36,7 +34,7 @@ export default class DOMManager extends ListWalker { return this.startTime; } - add(m: TimedMessage): void { + add(m: Message): void { switch (m.tp) { case "set_node_scroll": if (!this.nodeScrollManagers[ m.id ]) { @@ -104,8 +102,9 @@ export default class DOMManager extends ListWalker { if ((el instanceof HTMLStyleElement) && // TODO: correct ordering OR filter in tracker el.sheet && el.sheet.cssRules && - el.sheet.cssRules.length > 0) { - logger.log("Trying to insert child to style tag with virtual rules: ", this.nl[ parentID ], this.nl[ id ]); + el.sheet.cssRules.length > 0 && + el.innerText.trim().length === 0) { + logger.log("Trying to insert child to a style tag with virtual rules: ", this.nl[ parentID ], this.nl[ id ]); return; } @@ -183,6 +182,9 @@ export default class DOMManager extends ListWalker { } this.stylesManager.setStyleHandlers(node, value); } + if (node.namespaceURI === 'http://www.w3.org/2000/svg' && value.startsWith("url(")) { + value = "url(#" + (value.split("#")[1] ||")") + } try { node.setAttribute(name, value); } catch(e) { diff --git a/frontend/app/player/MessageDistributor/managers/ListWalker.ts b/frontend/app/player/MessageDistributor/managers/ListWalker.ts index 6283ff3ab..dcfe5cd96 100644 --- a/frontend/app/player/MessageDistributor/managers/ListWalker.ts +++ b/frontend/app/player/MessageDistributor/managers/ListWalker.ts @@ -1,4 +1,4 @@ -import type { Timed } from '../Timed'; +import type { Timed } from '../messages/timed'; export default class ListWalker { // Optimisation: #prop compiles to method that costs mor than strict property call. diff --git a/frontend/app/player/MessageDistributor/managers/MobXStateManager.ts b/frontend/app/player/MessageDistributor/managers/MobXStateManager.ts new file mode 100644 index 000000000..7a181fcf0 --- /dev/null +++ b/frontend/app/player/MessageDistributor/managers/MobXStateManager.ts @@ -0,0 +1,14 @@ +// import type { MobX } from '../messages'; +// import type { Timed } from '../Timed'; + +// import ListWalker from './ListWalker'; + +// type MobXTimed = MobX & Timed; + +// export default class MobXStateManager extends ListWalker { +// moveToLast(t: number) { +// super.moveApply(t, ) +// } + + +// } \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/managers/MouseManager.ts b/frontend/app/player/MessageDistributor/managers/MouseManager.ts index cd26c5b7f..ba174ae89 100644 --- a/frontend/app/player/MessageDistributor/managers/MouseManager.ts +++ b/frontend/app/player/MessageDistributor/managers/MouseManager.ts @@ -1,15 +1,12 @@ import type StatedScreen from '../StatedScreen'; import type { MouseMove } from '../messages'; -import type { Timed } from '../Timed'; import ListWalker from './ListWalker'; -type MouseMoveTimed = MouseMove & Timed; - const HOVER_CLASS = "-openreplay-hover"; const HOVER_CLASS_DEPR = "-asayer-hover"; -export default class MouseManager extends ListWalker { +export default class MouseManager extends ListWalker { private hoverElements: Array = []; constructor(private screen: StatedScreen) {super();} @@ -39,6 +36,7 @@ export default class MouseManager extends ListWalker { if (!!lastMouseMove){ // @ts-ignore TODO this.screen.cursor.move(lastMouseMove); + //window.getComputedStyle(this.screen.getCursorTarget()).cursor === 'pointer' // might nfluence performance though this.updateHover(); } } diff --git a/frontend/app/player/MessageDistributor/managers/PerformanceTrackManager.ts b/frontend/app/player/MessageDistributor/managers/PerformanceTrackManager.ts index 1b11813b2..4c756616e 100644 --- a/frontend/app/player/MessageDistributor/managers/PerformanceTrackManager.ts +++ b/frontend/app/player/MessageDistributor/managers/PerformanceTrackManager.ts @@ -1,11 +1,7 @@ import type { PerformanceTrack, SetPageVisibility } from '../messages'; -import type { Timed } from '../Timed'; import ListWalker from './ListWalker'; -type TimedPerformanceTrack = Timed & PerformanceTrack; -type TimedSetPageVisibility = Timed & SetPageVisibility; - export type PerformanceChartPoint = { time: number, usedHeap: number, @@ -15,7 +11,7 @@ export type PerformanceChartPoint = { nodesCount: number, } -export default class PerformanceTrackManager extends ListWalker { +export default class PerformanceTrackManager extends ListWalker { private chart: Array = []; private isHidden: boolean = false; private timeCorrection: number = 0; @@ -26,7 +22,7 @@ export default class PerformanceTrackManager extends ListWalker { +export default class StylesManager extends ListWalker { private linkLoadingCount: number = 0; private linkLoadPromises: Array> = []; private skipCSSLinks: Array = []; // should be common for all pages diff --git a/frontend/app/player/MessageDistributor/messages.ts b/frontend/app/player/MessageDistributor/messages.ts deleted file mode 100644 index bb391b9f4..000000000 --- a/frontend/app/player/MessageDistributor/messages.ts +++ /dev/null @@ -1,715 +0,0 @@ -// Auto-generated, do not edit - -import PrimitiveReader from './PrimitiveReader'; - -export const ID_TP_MAP = { - - 0: "timestamp", - 2: "session_disconnect", - 4: "set_page_location", - 5: "set_viewport_size", - 6: "set_viewport_scroll", - 7: "create_document", - 8: "create_element_node", - 9: "create_text_node", - 10: "move_node", - 11: "remove_node", - 12: "set_node_attribute", - 13: "remove_node_attribute", - 14: "set_node_data", - 15: "set_css_data", - 16: "set_node_scroll", - 18: "set_input_value", - 19: "set_input_checked", - 20: "mouse_move", - 22: "console_log", - 37: "css_insert_rule", - 38: "css_delete_rule", - 39: "fetch", - 40: "profiler", - 41: "o_table", - 44: "redux", - 45: "vuex", - 46: "mob_x", - 47: "ng_rx", - 48: "graph_ql", - 49: "performance_track", - 54: "connection_information", - 55: "set_page_visibility", - 59: "long_task", - 69: "mouse_click", - 70: "create_i_frame_document", - 90: "ios_session_start", - 93: "ios_custom_event", - 96: "ios_screen_changes", - 100: "ios_click_event", - 102: "ios_performance_event", - 103: "ios_log", - 105: "ios_network_call", -} as const; - - -export interface Timestamp { - tp: "timestamp", - timestamp: number, -} - -export interface SessionDisconnect { - tp: "session_disconnect", - timestamp: number, -} - -export interface SetPageLocation { - tp: "set_page_location", - url: string, - referrer: string, - navigationStart: number, -} - -export interface SetViewportSize { - tp: "set_viewport_size", - width: number, - height: number, -} - -export interface SetViewportScroll { - tp: "set_viewport_scroll", - x: number, - y: number, -} - -export interface CreateDocument { - tp: "create_document", - -} - -export interface CreateElementNode { - tp: "create_element_node", - id: number, - parentID: number, - index: number, - tag: string, - svg: boolean, -} - -export interface CreateTextNode { - tp: "create_text_node", - id: number, - parentID: number, - index: number, -} - -export interface MoveNode { - tp: "move_node", - id: number, - parentID: number, - index: number, -} - -export interface RemoveNode { - tp: "remove_node", - id: number, -} - -export interface SetNodeAttribute { - tp: "set_node_attribute", - id: number, - name: string, - value: string, -} - -export interface RemoveNodeAttribute { - tp: "remove_node_attribute", - id: number, - name: string, -} - -export interface SetNodeData { - tp: "set_node_data", - id: number, - data: string, -} - -export interface SetCssData { - tp: "set_css_data", - id: number, - data: string, -} - -export interface SetNodeScroll { - tp: "set_node_scroll", - id: number, - x: number, - y: number, -} - -export interface SetInputValue { - tp: "set_input_value", - id: number, - value: string, - mask: number, -} - -export interface SetInputChecked { - tp: "set_input_checked", - id: number, - checked: boolean, -} - -export interface MouseMove { - tp: "mouse_move", - x: number, - y: number, -} - -export interface ConsoleLog { - tp: "console_log", - level: string, - value: string, -} - -export interface CssInsertRule { - tp: "css_insert_rule", - id: number, - rule: string, - index: number, -} - -export interface CssDeleteRule { - tp: "css_delete_rule", - id: number, - index: number, -} - -export interface Fetch { - tp: "fetch", - method: string, - url: string, - request: string, - response: string, - status: number, - timestamp: number, - duration: number, -} - -export interface Profiler { - tp: "profiler", - name: string, - duration: number, - args: string, - result: string, -} - -export interface OTable { - tp: "o_table", - key: string, - value: string, -} - -export interface Redux { - tp: "redux", - action: string, - state: string, - duration: number, -} - -export interface Vuex { - tp: "vuex", - mutation: string, - state: string, -} - -export interface MobX { - tp: "mob_x", - type: string, - payload: string, -} - -export interface NgRx { - tp: "ng_rx", - action: string, - state: string, - duration: number, -} - -export interface GraphQl { - tp: "graph_ql", - operationKind: string, - operationName: string, - variables: string, - response: string, -} - -export interface PerformanceTrack { - tp: "performance_track", - frames: number, - ticks: number, - totalJSHeapSize: number, - usedJSHeapSize: number, -} - -export interface ConnectionInformation { - tp: "connection_information", - downlink: number, - type: string, -} - -export interface SetPageVisibility { - tp: "set_page_visibility", - hidden: boolean, -} - -export interface LongTask { - tp: "long_task", - timestamp: number, - duration: number, - context: number, - containerType: number, - containerSrc: string, - containerId: string, - containerName: string, -} - -export interface MouseClick { - tp: "mouse_click", - id: number, - hesitationTime: number, - label: string, - selector: string, -} - -export interface CreateIFrameDocument { - tp: "create_i_frame_document", - frameID: number, - id: number, -} - -export interface IosSessionStart { - tp: "ios_session_start", - timestamp: number, - projectID: number, - trackerVersion: string, - revID: string, - userUUID: string, - userOS: string, - userOSVersion: string, - userDevice: string, - userDeviceType: string, - userCountry: string, -} - -export interface IosCustomEvent { - tp: "ios_custom_event", - timestamp: number, - length: number, - name: string, - payload: string, -} - -export interface IosScreenChanges { - tp: "ios_screen_changes", - timestamp: number, - length: number, - x: number, - y: number, - width: number, - height: number, -} - -export interface IosClickEvent { - tp: "ios_click_event", - timestamp: number, - length: number, - label: string, - x: number, - y: number, -} - -export interface IosPerformanceEvent { - tp: "ios_performance_event", - timestamp: number, - length: number, - name: string, - value: number, -} - -export interface IosLog { - tp: "ios_log", - timestamp: number, - length: number, - severity: string, - content: string, -} - -export interface IosNetworkCall { - tp: "ios_network_call", - timestamp: number, - length: number, - duration: number, - headers: string, - body: string, - url: string, - success: boolean, - method: string, - status: number, -} - - -export type Message = Timestamp | SessionDisconnect | SetPageLocation | SetViewportSize | SetViewportScroll | CreateDocument | CreateElementNode | CreateTextNode | MoveNode | RemoveNode | SetNodeAttribute | RemoveNodeAttribute | SetNodeData | SetCssData | SetNodeScroll | SetInputValue | SetInputChecked | MouseMove | ConsoleLog | CssInsertRule | CssDeleteRule | Fetch | Profiler | OTable | Redux | Vuex | MobX | NgRx | GraphQl | PerformanceTrack | ConnectionInformation | SetPageVisibility | LongTask | MouseClick | CreateIFrameDocument | IosSessionStart | IosCustomEvent | IosScreenChanges | IosClickEvent | IosPerformanceEvent | IosLog | IosNetworkCall; - -export default function (r: PrimitiveReader): Message | null { - const tp = r.readUint() - switch (tp) { - - case 0: - return { - tp: ID_TP_MAP[0], - timestamp: r.readUint(), - }; - - case 2: - return { - tp: ID_TP_MAP[2], - timestamp: r.readUint(), - }; - - case 4: - return { - tp: ID_TP_MAP[4], - url: r.readString(), - referrer: r.readString(), - navigationStart: r.readUint(), - }; - - case 5: - return { - tp: ID_TP_MAP[5], - width: r.readUint(), - height: r.readUint(), - }; - - case 6: - return { - tp: ID_TP_MAP[6], - x: r.readInt(), - y: r.readInt(), - }; - - case 7: - return { - tp: ID_TP_MAP[7], - - }; - - case 8: - return { - tp: ID_TP_MAP[8], - id: r.readUint(), - parentID: r.readUint(), - index: r.readUint(), - tag: r.readString(), - svg: r.readBoolean(), - }; - - case 9: - return { - tp: ID_TP_MAP[9], - id: r.readUint(), - parentID: r.readUint(), - index: r.readUint(), - }; - - case 10: - return { - tp: ID_TP_MAP[10], - id: r.readUint(), - parentID: r.readUint(), - index: r.readUint(), - }; - - case 11: - return { - tp: ID_TP_MAP[11], - id: r.readUint(), - }; - - case 12: - return { - tp: ID_TP_MAP[12], - id: r.readUint(), - name: r.readString(), - value: r.readString(), - }; - - case 13: - return { - tp: ID_TP_MAP[13], - id: r.readUint(), - name: r.readString(), - }; - - case 14: - return { - tp: ID_TP_MAP[14], - id: r.readUint(), - data: r.readString(), - }; - - case 15: - return { - tp: ID_TP_MAP[15], - id: r.readUint(), - data: r.readString(), - }; - - case 16: - return { - tp: ID_TP_MAP[16], - id: r.readUint(), - x: r.readInt(), - y: r.readInt(), - }; - - case 18: - return { - tp: ID_TP_MAP[18], - id: r.readUint(), - value: r.readString(), - mask: r.readInt(), - }; - - case 19: - return { - tp: ID_TP_MAP[19], - id: r.readUint(), - checked: r.readBoolean(), - }; - - case 20: - return { - tp: ID_TP_MAP[20], - x: r.readUint(), - y: r.readUint(), - }; - - case 22: - return { - tp: ID_TP_MAP[22], - level: r.readString(), - value: r.readString(), - }; - - case 37: - return { - tp: ID_TP_MAP[37], - id: r.readUint(), - rule: r.readString(), - index: r.readUint(), - }; - - case 38: - return { - tp: ID_TP_MAP[38], - id: r.readUint(), - index: r.readUint(), - }; - - case 39: - return { - tp: ID_TP_MAP[39], - method: r.readString(), - url: r.readString(), - request: r.readString(), - response: r.readString(), - status: r.readUint(), - timestamp: r.readUint(), - duration: r.readUint(), - }; - - case 40: - return { - tp: ID_TP_MAP[40], - name: r.readString(), - duration: r.readUint(), - args: r.readString(), - result: r.readString(), - }; - - case 41: - return { - tp: ID_TP_MAP[41], - key: r.readString(), - value: r.readString(), - }; - - case 44: - return { - tp: ID_TP_MAP[44], - action: r.readString(), - state: r.readString(), - duration: r.readUint(), - }; - - case 45: - return { - tp: ID_TP_MAP[45], - mutation: r.readString(), - state: r.readString(), - }; - - case 46: - return { - tp: ID_TP_MAP[46], - type: r.readString(), - payload: r.readString(), - }; - - case 47: - return { - tp: ID_TP_MAP[47], - action: r.readString(), - state: r.readString(), - duration: r.readUint(), - }; - - case 48: - return { - tp: ID_TP_MAP[48], - operationKind: r.readString(), - operationName: r.readString(), - variables: r.readString(), - response: r.readString(), - }; - - case 49: - return { - tp: ID_TP_MAP[49], - frames: r.readInt(), - ticks: r.readInt(), - totalJSHeapSize: r.readUint(), - usedJSHeapSize: r.readUint(), - }; - - case 54: - return { - tp: ID_TP_MAP[54], - downlink: r.readUint(), - type: r.readString(), - }; - - case 55: - return { - tp: ID_TP_MAP[55], - hidden: r.readBoolean(), - }; - - case 59: - return { - tp: ID_TP_MAP[59], - timestamp: r.readUint(), - duration: r.readUint(), - context: r.readUint(), - containerType: r.readUint(), - containerSrc: r.readString(), - containerId: r.readString(), - containerName: r.readString(), - }; - - case 69: - return { - tp: ID_TP_MAP[69], - id: r.readUint(), - hesitationTime: r.readUint(), - label: r.readString(), - selector: r.readString(), - }; - - case 70: - return { - tp: ID_TP_MAP[70], - frameID: r.readUint(), - id: r.readUint(), - }; - - case 90: - return { - tp: ID_TP_MAP[90], - timestamp: r.readUint(), - projectID: r.readUint(), - trackerVersion: r.readString(), - revID: r.readString(), - userUUID: r.readString(), - userOS: r.readString(), - userOSVersion: r.readString(), - userDevice: r.readString(), - userDeviceType: r.readString(), - userCountry: r.readString(), - }; - - case 93: - return { - tp: ID_TP_MAP[93], - timestamp: r.readUint(), - length: r.readUint(), - name: r.readString(), - payload: r.readString(), - }; - - case 96: - return { - tp: ID_TP_MAP[96], - timestamp: r.readUint(), - length: r.readUint(), - x: r.readUint(), - y: r.readUint(), - width: r.readUint(), - height: r.readUint(), - }; - - case 100: - return { - tp: ID_TP_MAP[100], - timestamp: r.readUint(), - length: r.readUint(), - label: r.readString(), - x: r.readUint(), - y: r.readUint(), - }; - - case 102: - return { - tp: ID_TP_MAP[102], - timestamp: r.readUint(), - length: r.readUint(), - name: r.readString(), - value: r.readUint(), - }; - - case 103: - return { - tp: ID_TP_MAP[103], - timestamp: r.readUint(), - length: r.readUint(), - severity: r.readString(), - content: r.readString(), - }; - - case 105: - return { - tp: ID_TP_MAP[105], - timestamp: r.readUint(), - length: r.readUint(), - duration: r.readUint(), - headers: r.readString(), - body: r.readString(), - url: r.readString(), - success: r.readBoolean(), - method: r.readString(), - status: r.readUint(), - }; - - default: - throw new Error(`Unrecognizable message type: ${ tp }`) - return null; - } -} diff --git a/frontend/app/player/MessageDistributor/messages/JSONRawMessageReader.ts b/frontend/app/player/MessageDistributor/messages/JSONRawMessageReader.ts new file mode 100644 index 000000000..8143ae17c --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/JSONRawMessageReader.ts @@ -0,0 +1,18 @@ +import type { RawMessage } from './raw' + +import { TP_MAP } from './raw' + +export default class JSONRawMessageReader { + constructor(private messages: any[] = []){} + append(messages: any[]) { + this.messages = this.messages.concat(messages) + } + readMessage(): RawMessage | null { + const msg = this.messages.shift() + if (!msg) { return null } + msg.tp = TP_MAP[msg._id] + delete msg._id + return msg as RawMessage + } + +} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/messages/MFileReader.ts b/frontend/app/player/MessageDistributor/messages/MFileReader.ts new file mode 100644 index 000000000..0204259e5 --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/MFileReader.ts @@ -0,0 +1,68 @@ +import type { Message } from './message'; +import type { RawMessage } from './raw'; +import logger from 'App/logger'; +import RawMessageReader from './RawMessageReader'; + +// TODO: composition instead of inheritance +// needSkipMessage() and next() methods here use buf and p protected properties, +// which should be probably somehow incapsulated +export default class MFileReader extends RawMessageReader { + private pLastMessageID: number = 0; + private currentTime: number = 0; + public error: boolean = false; + constructor(data: Uint8Array, private readonly startTime: number) { + super(data); + } + + private needSkipMessage(): boolean { + if (this.p === 0) return false; + for (let i = 7; i >= 0; i--) { + if (this.buf[ this.p + i ] !== this.buf[ this.pLastMessageID + i ]) { + return this.buf[ this.p + i ] - this.buf[ this.pLastMessageID + i ] < 0; + } + } + return true; + } + + private readRawMessage(): RawMessage | null { + this.skip(8); + try { + return super.readMessage(); + } catch (e) { + this.error = true; + logger.error("Read message error:", e); + return null; + } + } + + hasNext():boolean { + return !this.error && this.hasNextByte(); + } + + next(): [ Message, number] | null { + if (!this.hasNext()) { + return null; + } + + while (this.needSkipMessage()) { + this.readRawMessage(); + } + this.pLastMessageID = this.p; + + const rMsg = this.readRawMessage(); + if (!rMsg) { + return null; + } + + if (rMsg.tp === "timestamp") { + this.currentTime = rMsg.timestamp - this.startTime; + } else { + const msg = Object.assign(rMsg, { + time: this.currentTime, + _index: this.pLastMessageID, + }) + return [msg, this.pLastMessageID]; + } + return null; + } +} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/messages/MStreamReader.ts b/frontend/app/player/MessageDistributor/messages/MStreamReader.ts new file mode 100644 index 000000000..1cc30dcec --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/MStreamReader.ts @@ -0,0 +1,69 @@ +import type { Message } from './message' +import type { + RawMessage, + RawSetNodeAttributeURLBased, + RawSetNodeAttribute, + RawSetCssDataURLBased, + RawSetCssData, + RawCssInsertRuleURLBased, + RawCssInsertRule, +} from './raw' +import RawMessageReader from './RawMessageReader' +import type { RawMessageReaderI } from './RawMessageReader' +import { resolveURL, resolveCSS } from './urlResolve' + + +const resolveMsg = { + "set_node_attribute_url_based": (msg: RawSetNodeAttributeURLBased): RawSetNodeAttribute => + ({ + ...msg, + value: msg.name === 'src' || msg.name === 'href' + ? resolveURL(msg.baseURL, msg.value) + : (msg.name === 'style' + ? resolveCSS(msg.baseURL, msg.value) + : msg.value + ), + tp: "set_node_attribute", + }), + "set_css_data_url_based": (msg: RawSetCssDataURLBased): RawSetCssData => + ({ + ...msg, + data: resolveCSS(msg.baseURL, msg.data), + tp: "set_css_data", + }), + "css_insert_rule_url_based": (msg: RawCssInsertRuleURLBased): RawCssInsertRule => + ({ + ...msg, + rule: resolveCSS(msg.baseURL, msg.rule), + tp: "css_insert_rule", + }) +} + +export default class MStreamReader { + constructor(private readonly r: RawMessageReaderI = new RawMessageReader()){} + + // append(buf: Uint8Array) { + // this.r.append(buf) + // } + + private t0: number = 0 + private t: number = 0 + private idx: number = 0 + readNext(): Message | null { + let msg = this.r.readMessage() + if (msg === null) { return null } + if (msg.tp === "timestamp" || msg.tp === "batch_meta") { + this.t0 = this.t0 || msg.timestamp + this.t = msg.timestamp - this.t0 + return this.readNext() + } + + // why typescript doesn't work here? + msg = (resolveMsg[msg.tp] || ((m:RawMessage)=>m))(msg) + + return Object.assign(msg, { + time: this.t, + _index: this.idx++, + }) + } +} \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/messages/PrimitiveReader.ts b/frontend/app/player/MessageDistributor/messages/PrimitiveReader.ts new file mode 100644 index 000000000..bc62bf653 --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/PrimitiveReader.ts @@ -0,0 +1,55 @@ +export default class PrimitiveReader { + protected p: number = 0 + constructor(protected buf: Uint8Array = new Uint8Array(0)) {} + + append(buf: Uint8Array) { + const newBuf = new Uint8Array(this.buf.length + buf.length) + newBuf.set(this.buf) + newBuf.set(buf, this.buf.length) + this.buf = newBuf + } + + hasNextByte(): boolean { + return this.p < this.buf.length + } + + readUint(): number | null { + let p = this.p, r = 0, s = 1, b + do { + if (p >= this.buf.length) { + return null + } + b = this.buf[ p++ ] + r += (b & 0x7F) * s + s *= 128; + } while (b >= 0x80) + this.p = p + return r; + } + + readInt(): number | null { + let u = this.readUint(); + if (u === null) { return u } + if (u % 2) { + u = (u + 1) / -2; + } else { + u = u / 2; + } + return u; + } + + readString(): string | null { + var l = this.readUint(); + if (l === null || this.p + l > this.buf.length) { return null } + return new TextDecoder().decode(this.buf.subarray(this.p, this.p+=l)); + } + + readBoolean(): boolean | null { + if (this.p >= this.buf.length) { return null } + return !!this.buf[this.p++]; + } + + skip(n: number) { + this.p += n; + } +} diff --git a/frontend/app/player/MessageDistributor/messages/RawMessageReader.ts b/frontend/app/player/MessageDistributor/messages/RawMessageReader.ts new file mode 100644 index 000000000..867d80755 --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/RawMessageReader.ts @@ -0,0 +1,758 @@ +// Auto-generated, do not edit + +import PrimitiveReader from './PrimitiveReader' +import type { RawMessage } from './raw' + +export interface RawMessageReaderI { + readMessage(): RawMessage | null +} + +export default class RawMessageReader extends PrimitiveReader { + readMessage(): RawMessage | null { + const p = this.p + const resetPointer = () => { + this.p = p + return null + } + + const tp = this.readUint() + if (tp === null) { return resetPointer() } + + switch (tp) { + + case 80: { + const pageNo = this.readUint(); if (pageNo === null) { return resetPointer() } + const firstIndex = this.readUint(); if (firstIndex === null) { return resetPointer() } + const timestamp = this.readInt(); if (timestamp === null) { return resetPointer() } + return { + tp: "batch_meta", + pageNo, + firstIndex, + timestamp, + }; + } + + case 0: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + return { + tp: "timestamp", + timestamp, + }; + } + + case 2: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + return { + tp: "session_disconnect", + timestamp, + }; + } + + case 4: { + const url = this.readString(); if (url === null) { return resetPointer() } + const referrer = this.readString(); if (referrer === null) { return resetPointer() } + const navigationStart = this.readUint(); if (navigationStart === null) { return resetPointer() } + return { + tp: "set_page_location", + url, + referrer, + navigationStart, + }; + } + + case 5: { + const width = this.readUint(); if (width === null) { return resetPointer() } + const height = this.readUint(); if (height === null) { return resetPointer() } + return { + tp: "set_viewport_size", + width, + height, + }; + } + + case 6: { + const x = this.readInt(); if (x === null) { return resetPointer() } + const y = this.readInt(); if (y === null) { return resetPointer() } + return { + tp: "set_viewport_scroll", + x, + y, + }; + } + + case 7: { + + return { + tp: "create_document", + + }; + } + + case 8: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const parentID = this.readUint(); if (parentID === null) { return resetPointer() } + const index = this.readUint(); if (index === null) { return resetPointer() } + const tag = this.readString(); if (tag === null) { return resetPointer() } + const svg = this.readBoolean(); if (svg === null) { return resetPointer() } + return { + tp: "create_element_node", + id, + parentID, + index, + tag, + svg, + }; + } + + case 9: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const parentID = this.readUint(); if (parentID === null) { return resetPointer() } + const index = this.readUint(); if (index === null) { return resetPointer() } + return { + tp: "create_text_node", + id, + parentID, + index, + }; + } + + case 10: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const parentID = this.readUint(); if (parentID === null) { return resetPointer() } + const index = this.readUint(); if (index === null) { return resetPointer() } + return { + tp: "move_node", + id, + parentID, + index, + }; + } + + case 11: { + const id = this.readUint(); if (id === null) { return resetPointer() } + return { + tp: "remove_node", + id, + }; + } + + case 12: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const name = this.readString(); if (name === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + return { + tp: "set_node_attribute", + id, + name, + value, + }; + } + + case 13: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const name = this.readString(); if (name === null) { return resetPointer() } + return { + tp: "remove_node_attribute", + id, + name, + }; + } + + case 14: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const data = this.readString(); if (data === null) { return resetPointer() } + return { + tp: "set_node_data", + id, + data, + }; + } + + case 15: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const data = this.readString(); if (data === null) { return resetPointer() } + return { + tp: "set_css_data", + id, + data, + }; + } + + case 16: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const x = this.readInt(); if (x === null) { return resetPointer() } + const y = this.readInt(); if (y === null) { return resetPointer() } + return { + tp: "set_node_scroll", + id, + x, + y, + }; + } + + case 17: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const label = this.readString(); if (label === null) { return resetPointer() } + return { + tp: "set_input_target", + id, + label, + }; + } + + case 18: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + const mask = this.readInt(); if (mask === null) { return resetPointer() } + return { + tp: "set_input_value", + id, + value, + mask, + }; + } + + case 19: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const checked = this.readBoolean(); if (checked === null) { return resetPointer() } + return { + tp: "set_input_checked", + id, + checked, + }; + } + + case 20: { + const x = this.readUint(); if (x === null) { return resetPointer() } + const y = this.readUint(); if (y === null) { return resetPointer() } + return { + tp: "mouse_move", + x, + y, + }; + } + + case 22: { + const level = this.readString(); if (level === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + return { + tp: "console_log", + level, + value, + }; + } + + case 23: { + const requestStart = this.readUint(); if (requestStart === null) { return resetPointer() } + const responseStart = this.readUint(); if (responseStart === null) { return resetPointer() } + const responseEnd = this.readUint(); if (responseEnd === null) { return resetPointer() } + const domContentLoadedEventStart = this.readUint(); if (domContentLoadedEventStart === null) { return resetPointer() } + const domContentLoadedEventEnd = this.readUint(); if (domContentLoadedEventEnd === null) { return resetPointer() } + const loadEventStart = this.readUint(); if (loadEventStart === null) { return resetPointer() } + const loadEventEnd = this.readUint(); if (loadEventEnd === null) { return resetPointer() } + const firstPaint = this.readUint(); if (firstPaint === null) { return resetPointer() } + const firstContentfulPaint = this.readUint(); if (firstContentfulPaint === null) { return resetPointer() } + return { + tp: "page_load_timing", + requestStart, + responseStart, + responseEnd, + domContentLoadedEventStart, + domContentLoadedEventEnd, + loadEventStart, + loadEventEnd, + firstPaint, + firstContentfulPaint, + }; + } + + case 24: { + const speedIndex = this.readUint(); if (speedIndex === null) { return resetPointer() } + const visuallyComplete = this.readUint(); if (visuallyComplete === null) { return resetPointer() } + const timeToInteractive = this.readUint(); if (timeToInteractive === null) { return resetPointer() } + return { + tp: "page_render_timing", + speedIndex, + visuallyComplete, + timeToInteractive, + }; + } + + case 25: { + const name = this.readString(); if (name === null) { return resetPointer() } + const message = this.readString(); if (message === null) { return resetPointer() } + const payload = this.readString(); if (payload === null) { return resetPointer() } + return { + tp: "js_exception", + name, + message, + payload, + }; + } + + case 27: { + const name = this.readString(); if (name === null) { return resetPointer() } + const payload = this.readString(); if (payload === null) { return resetPointer() } + return { + tp: "raw_custom_event", + name, + payload, + }; + } + + case 28: { + const id = this.readString(); if (id === null) { return resetPointer() } + return { + tp: "user_id", + id, + }; + } + + case 29: { + const id = this.readString(); if (id === null) { return resetPointer() } + return { + tp: "user_anonymous_id", + id, + }; + } + + case 30: { + const key = this.readString(); if (key === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + return { + tp: "metadata", + key, + value, + }; + } + + case 37: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const rule = this.readString(); if (rule === null) { return resetPointer() } + const index = this.readUint(); if (index === null) { return resetPointer() } + return { + tp: "css_insert_rule", + id, + rule, + index, + }; + } + + case 38: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const index = this.readUint(); if (index === null) { return resetPointer() } + return { + tp: "css_delete_rule", + id, + index, + }; + } + + case 39: { + const method = this.readString(); if (method === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const request = this.readString(); if (request === null) { return resetPointer() } + const response = this.readString(); if (response === null) { return resetPointer() } + const status = this.readUint(); if (status === null) { return resetPointer() } + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + return { + tp: "fetch", + method, + url, + request, + response, + status, + timestamp, + duration, + }; + } + + case 40: { + const name = this.readString(); if (name === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const args = this.readString(); if (args === null) { return resetPointer() } + const result = this.readString(); if (result === null) { return resetPointer() } + return { + tp: "profiler", + name, + duration, + args, + result, + }; + } + + case 41: { + const key = this.readString(); if (key === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + return { + tp: "o_table", + key, + value, + }; + } + + case 42: { + const type = this.readString(); if (type === null) { return resetPointer() } + return { + tp: "state_action", + type, + }; + } + + case 44: { + const action = this.readString(); if (action === null) { return resetPointer() } + const state = this.readString(); if (state === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + return { + tp: "redux", + action, + state, + duration, + }; + } + + case 45: { + const mutation = this.readString(); if (mutation === null) { return resetPointer() } + const state = this.readString(); if (state === null) { return resetPointer() } + return { + tp: "vuex", + mutation, + state, + }; + } + + case 46: { + const type = this.readString(); if (type === null) { return resetPointer() } + const payload = this.readString(); if (payload === null) { return resetPointer() } + return { + tp: "mob_x", + type, + payload, + }; + } + + case 47: { + const action = this.readString(); if (action === null) { return resetPointer() } + const state = this.readString(); if (state === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + return { + tp: "ng_rx", + action, + state, + duration, + }; + } + + case 48: { + const operationKind = this.readString(); if (operationKind === null) { return resetPointer() } + const operationName = this.readString(); if (operationName === null) { return resetPointer() } + const variables = this.readString(); if (variables === null) { return resetPointer() } + const response = this.readString(); if (response === null) { return resetPointer() } + return { + tp: "graph_ql", + operationKind, + operationName, + variables, + response, + }; + } + + case 49: { + const frames = this.readInt(); if (frames === null) { return resetPointer() } + const ticks = this.readInt(); if (ticks === null) { return resetPointer() } + const totalJSHeapSize = this.readUint(); if (totalJSHeapSize === null) { return resetPointer() } + const usedJSHeapSize = this.readUint(); if (usedJSHeapSize === null) { return resetPointer() } + return { + tp: "performance_track", + frames, + ticks, + totalJSHeapSize, + usedJSHeapSize, + }; + } + + case 53: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const ttfb = this.readUint(); if (ttfb === null) { return resetPointer() } + const headerSize = this.readUint(); if (headerSize === null) { return resetPointer() } + const encodedBodySize = this.readUint(); if (encodedBodySize === null) { return resetPointer() } + const decodedBodySize = this.readUint(); if (decodedBodySize === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const initiator = this.readString(); if (initiator === null) { return resetPointer() } + return { + tp: "resource_timing", + timestamp, + duration, + ttfb, + headerSize, + encodedBodySize, + decodedBodySize, + url, + initiator, + }; + } + + case 54: { + const downlink = this.readUint(); if (downlink === null) { return resetPointer() } + const type = this.readString(); if (type === null) { return resetPointer() } + return { + tp: "connection_information", + downlink, + type, + }; + } + + case 55: { + const hidden = this.readBoolean(); if (hidden === null) { return resetPointer() } + return { + tp: "set_page_visibility", + hidden, + }; + } + + case 59: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const context = this.readUint(); if (context === null) { return resetPointer() } + const containerType = this.readUint(); if (containerType === null) { return resetPointer() } + const containerSrc = this.readString(); if (containerSrc === null) { return resetPointer() } + const containerId = this.readString(); if (containerId === null) { return resetPointer() } + const containerName = this.readString(); if (containerName === null) { return resetPointer() } + return { + tp: "long_task", + timestamp, + duration, + context, + containerType, + containerSrc, + containerId, + containerName, + }; + } + + case 60: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const name = this.readString(); if (name === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + const baseURL = this.readString(); if (baseURL === null) { return resetPointer() } + return { + tp: "set_node_attribute_url_based", + id, + name, + value, + baseURL, + }; + } + + case 61: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const data = this.readString(); if (data === null) { return resetPointer() } + const baseURL = this.readString(); if (baseURL === null) { return resetPointer() } + return { + tp: "set_css_data_url_based", + id, + data, + baseURL, + }; + } + + case 63: { + const type = this.readString(); if (type === null) { return resetPointer() } + const value = this.readString(); if (value === null) { return resetPointer() } + return { + tp: "technical_info", + type, + value, + }; + } + + case 64: { + const name = this.readString(); if (name === null) { return resetPointer() } + const payload = this.readString(); if (payload === null) { return resetPointer() } + return { + tp: "custom_issue", + name, + payload, + }; + } + + case 65: { + + return { + tp: "page_close", + + }; + } + + case 67: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const rule = this.readString(); if (rule === null) { return resetPointer() } + const index = this.readUint(); if (index === null) { return resetPointer() } + const baseURL = this.readString(); if (baseURL === null) { return resetPointer() } + return { + tp: "css_insert_rule_url_based", + id, + rule, + index, + baseURL, + }; + } + + case 69: { + const id = this.readUint(); if (id === null) { return resetPointer() } + const hesitationTime = this.readUint(); if (hesitationTime === null) { return resetPointer() } + const label = this.readString(); if (label === null) { return resetPointer() } + const selector = this.readString(); if (selector === null) { return resetPointer() } + return { + tp: "mouse_click", + id, + hesitationTime, + label, + selector, + }; + } + + case 70: { + const frameID = this.readUint(); if (frameID === null) { return resetPointer() } + const id = this.readUint(); if (id === null) { return resetPointer() } + return { + tp: "create_i_frame_document", + frameID, + id, + }; + } + + case 90: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const projectID = this.readUint(); if (projectID === null) { return resetPointer() } + const trackerVersion = this.readString(); if (trackerVersion === null) { return resetPointer() } + const revID = this.readString(); if (revID === null) { return resetPointer() } + const userUUID = this.readString(); if (userUUID === null) { return resetPointer() } + const userOS = this.readString(); if (userOS === null) { return resetPointer() } + const userOSVersion = this.readString(); if (userOSVersion === null) { return resetPointer() } + const userDevice = this.readString(); if (userDevice === null) { return resetPointer() } + const userDeviceType = this.readString(); if (userDeviceType === null) { return resetPointer() } + const userCountry = this.readString(); if (userCountry === null) { return resetPointer() } + return { + tp: "ios_session_start", + timestamp, + projectID, + trackerVersion, + revID, + userUUID, + userOS, + userOSVersion, + userDevice, + userDeviceType, + userCountry, + }; + } + + case 93: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const length = this.readUint(); if (length === null) { return resetPointer() } + const name = this.readString(); if (name === null) { return resetPointer() } + const payload = this.readString(); if (payload === null) { return resetPointer() } + return { + tp: "ios_custom_event", + timestamp, + length, + name, + payload, + }; + } + + case 96: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const length = this.readUint(); if (length === null) { return resetPointer() } + const x = this.readUint(); if (x === null) { return resetPointer() } + const y = this.readUint(); if (y === null) { return resetPointer() } + const width = this.readUint(); if (width === null) { return resetPointer() } + const height = this.readUint(); if (height === null) { return resetPointer() } + return { + tp: "ios_screen_changes", + timestamp, + length, + x, + y, + width, + height, + }; + } + + case 100: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const length = this.readUint(); if (length === null) { return resetPointer() } + const label = this.readString(); if (label === null) { return resetPointer() } + const x = this.readUint(); if (x === null) { return resetPointer() } + const y = this.readUint(); if (y === null) { return resetPointer() } + return { + tp: "ios_click_event", + timestamp, + length, + label, + x, + y, + }; + } + + case 102: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const length = this.readUint(); if (length === null) { return resetPointer() } + const name = this.readString(); if (name === null) { return resetPointer() } + const value = this.readUint(); if (value === null) { return resetPointer() } + return { + tp: "ios_performance_event", + timestamp, + length, + name, + value, + }; + } + + case 103: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const length = this.readUint(); if (length === null) { return resetPointer() } + const severity = this.readString(); if (severity === null) { return resetPointer() } + const content = this.readString(); if (content === null) { return resetPointer() } + return { + tp: "ios_log", + timestamp, + length, + severity, + content, + }; + } + + case 105: { + const timestamp = this.readUint(); if (timestamp === null) { return resetPointer() } + const length = this.readUint(); if (length === null) { return resetPointer() } + const duration = this.readUint(); if (duration === null) { return resetPointer() } + const headers = this.readString(); if (headers === null) { return resetPointer() } + const body = this.readString(); if (body === null) { return resetPointer() } + const url = this.readString(); if (url === null) { return resetPointer() } + const success = this.readBoolean(); if (success === null) { return resetPointer() } + const method = this.readString(); if (method === null) { return resetPointer() } + const status = this.readUint(); if (status === null) { return resetPointer() } + return { + tp: "ios_network_call", + timestamp, + length, + duration, + headers, + body, + url, + success, + method, + status, + }; + } + + default: + throw new Error(`Unrecognizable message type: ${ tp }`) + return null; + } + } +} diff --git a/frontend/app/player/MessageDistributor/messages/index.ts b/frontend/app/player/MessageDistributor/messages/index.ts new file mode 100644 index 000000000..2619b58cd --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/index.ts @@ -0,0 +1 @@ +export * from './message' \ No newline at end of file diff --git a/frontend/app/player/MessageDistributor/messages/message.ts b/frontend/app/player/MessageDistributor/messages/message.ts new file mode 100644 index 000000000..1c21bbfd2 --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/message.ts @@ -0,0 +1,184 @@ +// Auto-generated, do not edit + +import type { Timed } from './timed' +import type { RawMessage } from './raw' +import type { RawBatchMeta, + RawTimestamp, + RawSessionDisconnect, + RawSetPageLocation, + RawSetViewportSize, + RawSetViewportScroll, + RawCreateDocument, + RawCreateElementNode, + RawCreateTextNode, + RawMoveNode, + RawRemoveNode, + RawSetNodeAttribute, + RawRemoveNodeAttribute, + RawSetNodeData, + RawSetCssData, + RawSetNodeScroll, + RawSetInputTarget, + RawSetInputValue, + RawSetInputChecked, + RawMouseMove, + RawConsoleLog, + RawPageLoadTiming, + RawPageRenderTiming, + RawJsException, + RawRawCustomEvent, + RawUserID, + RawUserAnonymousID, + RawMetadata, + RawCssInsertRule, + RawCssDeleteRule, + RawFetch, + RawProfiler, + RawOTable, + RawStateAction, + RawRedux, + RawVuex, + RawMobX, + RawNgRx, + RawGraphQl, + RawPerformanceTrack, + RawResourceTiming, + RawConnectionInformation, + RawSetPageVisibility, + RawLongTask, + RawSetNodeAttributeURLBased, + RawSetCssDataURLBased, + RawTechnicalInfo, + RawCustomIssue, + RawPageClose, + RawCssInsertRuleURLBased, + RawMouseClick, + RawCreateIFrameDocument, + RawIosSessionStart, + RawIosCustomEvent, + RawIosScreenChanges, + RawIosClickEvent, + RawIosPerformanceEvent, + RawIosLog, + RawIosNetworkCall, } from './raw' + +export type Message = RawMessage & Timed + + +export type BatchMeta = RawBatchMeta & Timed + +export type Timestamp = RawTimestamp & Timed + +export type SessionDisconnect = RawSessionDisconnect & Timed + +export type SetPageLocation = RawSetPageLocation & Timed + +export type SetViewportSize = RawSetViewportSize & Timed + +export type SetViewportScroll = RawSetViewportScroll & Timed + +export type CreateDocument = RawCreateDocument & Timed + +export type CreateElementNode = RawCreateElementNode & Timed + +export type CreateTextNode = RawCreateTextNode & Timed + +export type MoveNode = RawMoveNode & Timed + +export type RemoveNode = RawRemoveNode & Timed + +export type SetNodeAttribute = RawSetNodeAttribute & Timed + +export type RemoveNodeAttribute = RawRemoveNodeAttribute & Timed + +export type SetNodeData = RawSetNodeData & Timed + +export type SetCssData = RawSetCssData & Timed + +export type SetNodeScroll = RawSetNodeScroll & Timed + +export type SetInputTarget = RawSetInputTarget & Timed + +export type SetInputValue = RawSetInputValue & Timed + +export type SetInputChecked = RawSetInputChecked & Timed + +export type MouseMove = RawMouseMove & Timed + +export type ConsoleLog = RawConsoleLog & Timed + +export type PageLoadTiming = RawPageLoadTiming & Timed + +export type PageRenderTiming = RawPageRenderTiming & Timed + +export type JsException = RawJsException & Timed + +export type RawCustomEvent = RawRawCustomEvent & Timed + +export type UserID = RawUserID & Timed + +export type UserAnonymousID = RawUserAnonymousID & Timed + +export type Metadata = RawMetadata & Timed + +export type CssInsertRule = RawCssInsertRule & Timed + +export type CssDeleteRule = RawCssDeleteRule & Timed + +export type Fetch = RawFetch & Timed + +export type Profiler = RawProfiler & Timed + +export type OTable = RawOTable & Timed + +export type StateAction = RawStateAction & Timed + +export type Redux = RawRedux & Timed + +export type Vuex = RawVuex & Timed + +export type MobX = RawMobX & Timed + +export type NgRx = RawNgRx & Timed + +export type GraphQl = RawGraphQl & Timed + +export type PerformanceTrack = RawPerformanceTrack & Timed + +export type ResourceTiming = RawResourceTiming & Timed + +export type ConnectionInformation = RawConnectionInformation & Timed + +export type SetPageVisibility = RawSetPageVisibility & Timed + +export type LongTask = RawLongTask & Timed + +export type SetNodeAttributeURLBased = RawSetNodeAttributeURLBased & Timed + +export type SetCssDataURLBased = RawSetCssDataURLBased & Timed + +export type TechnicalInfo = RawTechnicalInfo & Timed + +export type CustomIssue = RawCustomIssue & Timed + +export type PageClose = RawPageClose & Timed + +export type CssInsertRuleURLBased = RawCssInsertRuleURLBased & Timed + +export type MouseClick = RawMouseClick & Timed + +export type CreateIFrameDocument = RawCreateIFrameDocument & Timed + +export type IosSessionStart = RawIosSessionStart & Timed + +export type IosCustomEvent = RawIosCustomEvent & Timed + +export type IosScreenChanges = RawIosScreenChanges & Timed + +export type IosClickEvent = RawIosClickEvent & Timed + +export type IosPerformanceEvent = RawIosPerformanceEvent & Timed + +export type IosLog = RawIosLog & Timed + +export type IosNetworkCall = RawIosNetworkCall & Timed diff --git a/frontend/app/player/MessageDistributor/messages/raw.ts b/frontend/app/player/MessageDistributor/messages/raw.ts new file mode 100644 index 000000000..e86181b2d --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/raw.ts @@ -0,0 +1,491 @@ +// Auto-generated, do not edit + +export const TP_MAP = { + 80: "batch_meta", + 0: "timestamp", + 2: "session_disconnect", + 4: "set_page_location", + 5: "set_viewport_size", + 6: "set_viewport_scroll", + 7: "create_document", + 8: "create_element_node", + 9: "create_text_node", + 10: "move_node", + 11: "remove_node", + 12: "set_node_attribute", + 13: "remove_node_attribute", + 14: "set_node_data", + 15: "set_css_data", + 16: "set_node_scroll", + 17: "set_input_target", + 18: "set_input_value", + 19: "set_input_checked", + 20: "mouse_move", + 22: "console_log", + 23: "page_load_timing", + 24: "page_render_timing", + 25: "js_exception", + 27: "raw_custom_event", + 28: "user_id", + 29: "user_anonymous_id", + 30: "metadata", + 37: "css_insert_rule", + 38: "css_delete_rule", + 39: "fetch", + 40: "profiler", + 41: "o_table", + 42: "state_action", + 44: "redux", + 45: "vuex", + 46: "mob_x", + 47: "ng_rx", + 48: "graph_ql", + 49: "performance_track", + 53: "resource_timing", + 54: "connection_information", + 55: "set_page_visibility", + 59: "long_task", + 60: "set_node_attribute_url_based", + 61: "set_css_data_url_based", + 63: "technical_info", + 64: "custom_issue", + 65: "page_close", + 67: "css_insert_rule_url_based", + 69: "mouse_click", + 70: "create_i_frame_document", + 90: "ios_session_start", + 93: "ios_custom_event", + 96: "ios_screen_changes", + 100: "ios_click_event", + 102: "ios_performance_event", + 103: "ios_log", + 105: "ios_network_call", +} + + +export interface RawBatchMeta { + tp: "batch_meta", + pageNo: number, + firstIndex: number, + timestamp: number, +} + +export interface RawTimestamp { + tp: "timestamp", + timestamp: number, +} + +export interface RawSessionDisconnect { + tp: "session_disconnect", + timestamp: number, +} + +export interface RawSetPageLocation { + tp: "set_page_location", + url: string, + referrer: string, + navigationStart: number, +} + +export interface RawSetViewportSize { + tp: "set_viewport_size", + width: number, + height: number, +} + +export interface RawSetViewportScroll { + tp: "set_viewport_scroll", + x: number, + y: number, +} + +export interface RawCreateDocument { + tp: "create_document", + +} + +export interface RawCreateElementNode { + tp: "create_element_node", + id: number, + parentID: number, + index: number, + tag: string, + svg: boolean, +} + +export interface RawCreateTextNode { + tp: "create_text_node", + id: number, + parentID: number, + index: number, +} + +export interface RawMoveNode { + tp: "move_node", + id: number, + parentID: number, + index: number, +} + +export interface RawRemoveNode { + tp: "remove_node", + id: number, +} + +export interface RawSetNodeAttribute { + tp: "set_node_attribute", + id: number, + name: string, + value: string, +} + +export interface RawRemoveNodeAttribute { + tp: "remove_node_attribute", + id: number, + name: string, +} + +export interface RawSetNodeData { + tp: "set_node_data", + id: number, + data: string, +} + +export interface RawSetCssData { + tp: "set_css_data", + id: number, + data: string, +} + +export interface RawSetNodeScroll { + tp: "set_node_scroll", + id: number, + x: number, + y: number, +} + +export interface RawSetInputTarget { + tp: "set_input_target", + id: number, + label: string, +} + +export interface RawSetInputValue { + tp: "set_input_value", + id: number, + value: string, + mask: number, +} + +export interface RawSetInputChecked { + tp: "set_input_checked", + id: number, + checked: boolean, +} + +export interface RawMouseMove { + tp: "mouse_move", + x: number, + y: number, +} + +export interface RawConsoleLog { + tp: "console_log", + level: string, + value: string, +} + +export interface RawPageLoadTiming { + tp: "page_load_timing", + requestStart: number, + responseStart: number, + responseEnd: number, + domContentLoadedEventStart: number, + domContentLoadedEventEnd: number, + loadEventStart: number, + loadEventEnd: number, + firstPaint: number, + firstContentfulPaint: number, +} + +export interface RawPageRenderTiming { + tp: "page_render_timing", + speedIndex: number, + visuallyComplete: number, + timeToInteractive: number, +} + +export interface RawJsException { + tp: "js_exception", + name: string, + message: string, + payload: string, +} + +export interface RawRawCustomEvent { + tp: "raw_custom_event", + name: string, + payload: string, +} + +export interface RawUserID { + tp: "user_id", + id: string, +} + +export interface RawUserAnonymousID { + tp: "user_anonymous_id", + id: string, +} + +export interface RawMetadata { + tp: "metadata", + key: string, + value: string, +} + +export interface RawCssInsertRule { + tp: "css_insert_rule", + id: number, + rule: string, + index: number, +} + +export interface RawCssDeleteRule { + tp: "css_delete_rule", + id: number, + index: number, +} + +export interface RawFetch { + tp: "fetch", + method: string, + url: string, + request: string, + response: string, + status: number, + timestamp: number, + duration: number, +} + +export interface RawProfiler { + tp: "profiler", + name: string, + duration: number, + args: string, + result: string, +} + +export interface RawOTable { + tp: "o_table", + key: string, + value: string, +} + +export interface RawStateAction { + tp: "state_action", + type: string, +} + +export interface RawRedux { + tp: "redux", + action: string, + state: string, + duration: number, +} + +export interface RawVuex { + tp: "vuex", + mutation: string, + state: string, +} + +export interface RawMobX { + tp: "mob_x", + type: string, + payload: string, +} + +export interface RawNgRx { + tp: "ng_rx", + action: string, + state: string, + duration: number, +} + +export interface RawGraphQl { + tp: "graph_ql", + operationKind: string, + operationName: string, + variables: string, + response: string, +} + +export interface RawPerformanceTrack { + tp: "performance_track", + frames: number, + ticks: number, + totalJSHeapSize: number, + usedJSHeapSize: number, +} + +export interface RawResourceTiming { + tp: "resource_timing", + timestamp: number, + duration: number, + ttfb: number, + headerSize: number, + encodedBodySize: number, + decodedBodySize: number, + url: string, + initiator: string, +} + +export interface RawConnectionInformation { + tp: "connection_information", + downlink: number, + type: string, +} + +export interface RawSetPageVisibility { + tp: "set_page_visibility", + hidden: boolean, +} + +export interface RawLongTask { + tp: "long_task", + timestamp: number, + duration: number, + context: number, + containerType: number, + containerSrc: string, + containerId: string, + containerName: string, +} + +export interface RawSetNodeAttributeURLBased { + tp: "set_node_attribute_url_based", + id: number, + name: string, + value: string, + baseURL: string, +} + +export interface RawSetCssDataURLBased { + tp: "set_css_data_url_based", + id: number, + data: string, + baseURL: string, +} + +export interface RawTechnicalInfo { + tp: "technical_info", + type: string, + value: string, +} + +export interface RawCustomIssue { + tp: "custom_issue", + name: string, + payload: string, +} + +export interface RawPageClose { + tp: "page_close", + +} + +export interface RawCssInsertRuleURLBased { + tp: "css_insert_rule_url_based", + id: number, + rule: string, + index: number, + baseURL: string, +} + +export interface RawMouseClick { + tp: "mouse_click", + id: number, + hesitationTime: number, + label: string, + selector: string, +} + +export interface RawCreateIFrameDocument { + tp: "create_i_frame_document", + frameID: number, + id: number, +} + +export interface RawIosSessionStart { + tp: "ios_session_start", + timestamp: number, + projectID: number, + trackerVersion: string, + revID: string, + userUUID: string, + userOS: string, + userOSVersion: string, + userDevice: string, + userDeviceType: string, + userCountry: string, +} + +export interface RawIosCustomEvent { + tp: "ios_custom_event", + timestamp: number, + length: number, + name: string, + payload: string, +} + +export interface RawIosScreenChanges { + tp: "ios_screen_changes", + timestamp: number, + length: number, + x: number, + y: number, + width: number, + height: number, +} + +export interface RawIosClickEvent { + tp: "ios_click_event", + timestamp: number, + length: number, + label: string, + x: number, + y: number, +} + +export interface RawIosPerformanceEvent { + tp: "ios_performance_event", + timestamp: number, + length: number, + name: string, + value: number, +} + +export interface RawIosLog { + tp: "ios_log", + timestamp: number, + length: number, + severity: string, + content: string, +} + +export interface RawIosNetworkCall { + tp: "ios_network_call", + timestamp: number, + length: number, + duration: number, + headers: string, + body: string, + url: string, + success: boolean, + method: string, + status: number, +} + + +export type RawMessage = RawBatchMeta | RawTimestamp | RawSessionDisconnect | RawSetPageLocation | RawSetViewportSize | RawSetViewportScroll | RawCreateDocument | RawCreateElementNode | RawCreateTextNode | RawMoveNode | RawRemoveNode | RawSetNodeAttribute | RawRemoveNodeAttribute | RawSetNodeData | RawSetCssData | RawSetNodeScroll | RawSetInputTarget | RawSetInputValue | RawSetInputChecked | RawMouseMove | RawConsoleLog | RawPageLoadTiming | RawPageRenderTiming | RawJsException | RawRawCustomEvent | RawUserID | RawUserAnonymousID | RawMetadata | RawCssInsertRule | RawCssDeleteRule | RawFetch | RawProfiler | RawOTable | RawStateAction | RawRedux | RawVuex | RawMobX | RawNgRx | RawGraphQl | RawPerformanceTrack | RawResourceTiming | RawConnectionInformation | RawSetPageVisibility | RawLongTask | RawSetNodeAttributeURLBased | RawSetCssDataURLBased | RawTechnicalInfo | RawCustomIssue | RawPageClose | RawCssInsertRuleURLBased | RawMouseClick | RawCreateIFrameDocument | RawIosSessionStart | RawIosCustomEvent | RawIosScreenChanges | RawIosClickEvent | RawIosPerformanceEvent | RawIosLog | RawIosNetworkCall; diff --git a/frontend/app/player/MessageDistributor/messages/timed.ts b/frontend/app/player/MessageDistributor/messages/timed.ts new file mode 100644 index 000000000..2dd4cc707 --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/timed.ts @@ -0,0 +1 @@ +export interface Timed { readonly time: number }; diff --git a/frontend/app/player/MessageDistributor/messages/urlResolve.ts b/frontend/app/player/MessageDistributor/messages/urlResolve.ts new file mode 100644 index 000000000..b80ff4f9a --- /dev/null +++ b/frontend/app/player/MessageDistributor/messages/urlResolve.ts @@ -0,0 +1,57 @@ +export function resolveURL(baseURL: string, relURL: string): string { + if (relURL.startsWith('#') || relURL === "") { + return relURL; + } + return new URL(relURL, baseURL).toString(); +} + + +var match = /bar/.exec("foobar"); +const re1 = /url\(("[^"]*"|'[^']*'|[^)]*)\)/g +const re2 = /@import "(.*?)"/g +function cssUrlsIndex(css: string): Array<[number, number]> { + const idxs: Array<[number, number]> = []; + const i1 = css.matchAll(re1); + // @ts-ignore + for (let m of i1) { + // @ts-ignore + const s: number = m.index + m[0].indexOf(m[1]); + const e: number = s + m[1].length; + idxs.push([s, e]); + } + const i2 = css.matchAll(re2); + // @ts-ignore + for (let m of i2) { + // @ts-ignore + const s = m.index + m[0].indexOf(m[1]); + const e = s + m[1].length; + idxs.push([s, e]) + } + return idxs; +} +function unquote(str: string): [string, string] { + str = str.trim(); + if (str.length <= 2) { + return [str, ""] + } + if (str[0] == '"' && str[str.length-1] == '"') { + return [ str.substring(1, str.length-1), "\""]; + } + if (str[0] == '\'' && str[str.length-1] == '\'') { + return [ str.substring(1, str.length-1), "'" ]; + } + return [str, ""] +} +function rewriteCSSLinks(css: string, rewriter: (rawurl: string) => string): string { + for (let idx of cssUrlsIndex(css)) { + const f = idx[0] + const t = idx[1] + const [ rawurl, q ] = unquote(css.substring(f, t)); + css = css.substring(0,f) + q + rewriter(rawurl) + q + css.substring(t); + } + return css +} + +export function resolveCSS(baseURL: string, css: string): string { + return rewriteCSSLinks(css, rawurl => resolveURL(baseURL, rawurl)); +} \ No newline at end of file diff --git a/frontend/app/player/ios/Parser.ts b/frontend/app/player/ios/Parser.ts index f202e9306..15b750df6 100644 --- a/frontend/app/player/ios/Parser.ts +++ b/frontend/app/player/ios/Parser.ts @@ -1,12 +1,11 @@ -import readMessage from '../MessageDistributor/messages'; -import PrimitiveReader from '../MessageDistributor/PrimitiveReader'; +import RawMessageReader from '../MessageDistributor/messages/RawMessageReader'; export default class Parser { - private reader: PrimitiveReader + private reader: RawMessageReader private error: boolean = false constructor(byteArray) { - this.reader = new PrimitiveReader(byteArray) + this.reader = new RawMessageReader(byteArray) } parseEach(cb) { @@ -19,12 +18,12 @@ export default class Parser { } hasNext() { - return !this.error && this.reader.hasNext(); + return !this.error && this.reader.hasNextByte(); } next() { try { - return readMessage(this.reader) + return this.reader.readMessage() } catch(e) { console.warn(e) this.error = true From 03ea03aba27d54211e75199ff81fa63ffc331f0b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 11 Jan 2022 11:05:50 +0100 Subject: [PATCH 077/206] feat(alerts): refactored alerts feat(alerts): fixed decimal precision feat(alerts): configurable cron-interval --- api/app_alerts.py | 4 +- api/build_alerts.sh | 4 +- api/chalicelib/core/alerts_listener.py | 27 +++ api/chalicelib/core/alerts_processor.py | 29 +-- ee/api/.gitignore | 5 +- ee/api/chalicelib/core/alerts_listener.py | 27 +++ ee/api/chalicelib/core/alerts_processor.py | 265 --------------------- 7 files changed, 66 insertions(+), 295 deletions(-) create mode 100644 api/chalicelib/core/alerts_listener.py create mode 100644 ee/api/chalicelib/core/alerts_listener.py delete mode 100644 ee/api/chalicelib/core/alerts_processor.py diff --git a/api/app_alerts.py b/api/app_alerts.py index 0cba8986a..60e1b7e48 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -1,6 +1,7 @@ import logging from apscheduler.schedulers.asyncio import AsyncIOScheduler +from decouple import config from fastapi import FastAPI from chalicelib.core import alerts_processor @@ -16,7 +17,8 @@ async def root(): app.schedule = AsyncIOScheduler() app.schedule.start() -app.schedule.add_job(id="alerts_processor", **{"func": alerts_processor.process, "trigger": "interval", "minutes": 5, +app.schedule.add_job(id="alerts_processor", **{"func": alerts_processor.process, "trigger": "interval", + "minutes": config("ALERTS_INTERVAL", cast=int, default=5), "misfire_grace_time": 20}) for job in app.schedule.get_jobs(): diff --git a/api/build_alerts.sh b/api/build_alerts.sh index bfe9c22a1..83fac6f83 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -15,7 +15,7 @@ function make_submodule() { mkdir -p ./alerts/chalicelib/ cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ mkdir -p ./alerts/chalicelib/core/ - cp -R ./chalicelib/core/{__init__,alerts_processor,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,assist,events_ios,sessions_mobs,errors,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ + cp -R ./chalicelib/core/{__init__,alerts_processor,alerts_listener,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,assist,events_ios,sessions_mobs,errors,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ mkdir -p ./alerts/chalicelib/utils/ cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,email_helper,email_handler,smtp,s3,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part @@ -27,7 +27,7 @@ function make_submodule() { mkdir -p ./alerts/chalicelib/ cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ mkdir -p ./alerts/chalicelib/core/ - cp -R ./chalicelib/core/{__init__,alerts_processor,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,events_ios,sessions_mobs,errors,dashboard,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ + cp -R ./chalicelib/core/{__init__,alerts_processor,alerts_listener,sessions,events,issues,sessions_metas,metadata,projects,users,authorizers,tenants,roles,assist,events_ios,sessions_mobs,errors,dashboard,sourcemaps,sourcemaps_parser,resources,performance_event,alerts,notifications,slack,collaboration_slack,webhook}.py ./alerts/chalicelib/core/ mkdir -p ./alerts/chalicelib/utils/ cp -R ./chalicelib/utils/{__init__,TimeUTC,pg_client,helper,event_filter_definition,dev,SAML2_helper,email_helper,email_handler,smtp,s3,args_transformer,ch_client,metrics_helper}.py ./alerts/chalicelib/utils/ # -- end of generated part diff --git a/api/chalicelib/core/alerts_listener.py b/api/chalicelib/core/alerts_listener.py new file mode 100644 index 000000000..419f0326d --- /dev/null +++ b/api/chalicelib/core/alerts_listener.py @@ -0,0 +1,27 @@ +from chalicelib.utils import pg_client, helper + + +def get_all_alerts(): + with pg_client.PostgresClient(long_query=True) as cur: + query = """SELECT -1 AS tenant_id, + alert_id, + project_id, + detection_method, + query, + options, + (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, + alerts.name, + alerts.series_id, + filter + FROM public.alerts + LEFT JOIN metric_series USING (series_id) + INNER JOIN projects USING (project_id) + WHERE alerts.deleted_at ISNULL + AND alerts.active + AND projects.active + AND projects.deleted_at ISNULL + AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) + ORDER BY alerts.created_at;""" + cur.execute(query=query) + all_alerts = helper.list_to_camel_case(cur.fetchall()) + return all_alerts diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 281064abf..43f57a96b 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -1,6 +1,7 @@ import schemas +from chalicelib.core import alerts_listener from chalicelib.core import sessions, alerts -from chalicelib.utils import pg_client, helper +from chalicelib.utils import pg_client from chalicelib.utils.TimeUTC import TimeUTC LeftToDb = { @@ -195,29 +196,7 @@ def Build(a): def process(): notifications = [] - with pg_client.PostgresClient(long_query=True) as cur: - query = """SELECT -1 AS tenant_id, - alert_id, - project_id, - detection_method, - query, - options, - (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, - alerts.name, - alerts.series_id, - filter - FROM public.alerts - LEFT JOIN metric_series USING (series_id) - INNER JOIN projects USING (project_id) - WHERE alerts.deleted_at ISNULL - AND alerts.active - AND projects.active - AND projects.deleted_at ISNULL - AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) - ORDER BY alerts.created_at;""" - cur.execute(query=query) - all_alerts = helper.list_to_camel_case(cur.fetchall()) - + all_alerts = alerts_listener.get_all_alerts() with pg_client.PostgresClient() as cur: for alert in all_alerts: if can_check(alert): @@ -235,7 +214,7 @@ def process(): "alertId": alert["alertId"], "tenantId": alert["tenantId"], "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {result['value']} ({alert['query']['operator']} {alert['query']['right']}).", + "description": f"has been triggered, {alert['query']['left']} = {round(result['value'], 2)} ({alert['query']['operator']} {alert['query']['right']}).", "buttonText": "Check metrics for more details", "buttonUrl": f"/{alert['projectId']}/metrics", "imageUrl": None, diff --git a/ee/api/.gitignore b/ee/api/.gitignore index cfd351eea..8afea0ab6 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -178,6 +178,7 @@ README/* Pipfile /chalicelib/core/alerts.py +/chalicelib/core/alerts_processor.py /chalicelib/core/announcements.py /chalicelib/blueprints/bp_app_api.py /chalicelib/blueprints/bp_core.py @@ -261,5 +262,5 @@ Pipfile /chalicelib/core/custom_metrics.py /chalicelib/core/performance_event.py /chalicelib/core/saved_search.py -/requirements_alerts.txt -/app_alerts.py \ No newline at end of file +/app_alerts.py +/build_alerts.sh diff --git a/ee/api/chalicelib/core/alerts_listener.py b/ee/api/chalicelib/core/alerts_listener.py new file mode 100644 index 000000000..40241f51e --- /dev/null +++ b/ee/api/chalicelib/core/alerts_listener.py @@ -0,0 +1,27 @@ +from chalicelib.utils import pg_client, helper + + +def get_all_alerts(): + with pg_client.PostgresClient(long_query=True) as cur: + query = """SELECT tenant_id, + alert_id, + project_id, + detection_method, + query, + options, + (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, + alerts.name, + alerts.series_id, + filter + FROM public.alerts + LEFT JOIN metric_series USING (series_id) + INNER JOIN projects USING (project_id) + WHERE alerts.deleted_at ISNULL + AND alerts.active + AND projects.active + AND projects.deleted_at ISNULL + AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) + ORDER BY alerts.created_at;""" + cur.execute(query=query) + all_alerts = helper.list_to_camel_case(cur.fetchall()) + return all_alerts diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py deleted file mode 100644 index 3bb6b202f..000000000 --- a/ee/api/chalicelib/core/alerts_processor.py +++ /dev/null @@ -1,265 +0,0 @@ -import schemas -from chalicelib.core import sessions, alerts -from chalicelib.utils import pg_client, helper -from chalicelib.utils.TimeUTC import TimeUTC - -LeftToDb = { - schemas.AlertColumn.performance__dom_content_loaded__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", - "formula": "COALESCE(AVG(NULLIF(dom_content_loaded_time ,0)),0)"}, - schemas.AlertColumn.performance__first_meaningful_paint__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", - "formula": "COALESCE(AVG(NULLIF(first_contentful_paint_time,0)),0)"}, - schemas.AlertColumn.performance__page_load_time__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(load_time ,0))"}, - schemas.AlertColumn.performance__dom_build_time__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(dom_building_time,0))"}, - schemas.AlertColumn.performance__speed_index__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", "formula": "AVG(NULLIF(speed_index,0))"}, - schemas.AlertColumn.performance__page_response_time__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(response_time,0))"}, - schemas.AlertColumn.performance__ttfb__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(first_paint_time,0))"}, - schemas.AlertColumn.performance__time_to_render__average: { - "table": "events.pages INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(visually_complete,0))"}, - schemas.AlertColumn.performance__image_load_time__average: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='img'"}, - schemas.AlertColumn.performance__request_load_time__average: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))", "condition": "type='fetch'"}, - schemas.AlertColumn.resources__load_time__average: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "AVG(NULLIF(resources.duration,0))"}, - schemas.AlertColumn.resources__missing__count: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT url_hostpath)", "condition": "success= FALSE"}, - schemas.AlertColumn.errors__4xx_5xx__count: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", "formula": "COUNT(session_id)", - "condition": "status/100!=2"}, - schemas.AlertColumn.errors__4xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=4"}, - schemas.AlertColumn.errors__5xx__count: {"table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(session_id)", "condition": "status/100=5"}, - schemas.AlertColumn.errors__javascript__impacted_sessions__count: { - "table": "events.resources INNER JOIN public.sessions USING(session_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "success= FALSE AND type='script'"}, - schemas.AlertColumn.performance__crashes__count: { - "table": "(SELECT *, start_ts AS timestamp FROM public.sessions WHERE errors_count > 0) AS sessions", - "formula": "COUNT(DISTINCT session_id)", "condition": "errors_count > 0"}, - schemas.AlertColumn.errors__javascript__count: { - "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "source='js_exception'", "joinSessions": False}, - schemas.AlertColumn.errors__backend__count: { - "table": "events.errors INNER JOIN public.errors AS m_errors USING (error_id)", - "formula": "COUNT(DISTINCT session_id)", "condition": "source!='js_exception'", "joinSessions": False}, -} - -# This is the frequency of execution for each threshold -TimeInterval = { - 15: 3, - 30: 5, - 60: 10, - 120: 20, - 240: 30, - 1440: 60, -} - - -def can_check(a) -> bool: - now = TimeUTC.now() - - repetitionBase = a["options"]["currentPeriod"] \ - if a["detectionMethod"] == schemas.AlertDetectionMethod.change \ - and a["options"]["currentPeriod"] > a["options"]["previousPeriod"] \ - else a["options"]["previousPeriod"] - - if TimeInterval.get(repetitionBase) is None: - print(f"repetitionBase: {repetitionBase} NOT FOUND") - return False - - return (a["options"]["renotifyInterval"] <= 0 or - a["options"].get("lastNotification") is None or - a["options"]["lastNotification"] <= 0 or - ((now - a["options"]["lastNotification"]) > a["options"]["renotifyInterval"] * 60 * 1000)) \ - and ((now - a["createdAt"]) % (TimeInterval[repetitionBase] * 60 * 1000)) < 60 * 1000 - - -def Build(a): - params = {"project_id": a["projectId"]} - full_args = {} - j_s = True - if a["seriesId"] is not None: - a["filter"]["sort"] = "session_id" - a["filter"]["order"] = "DESC" - a["filter"]["startDate"] = -1 - a["filter"]["endDate"] = TimeUTC.now() - full_args, query_part, sort = sessions.search_query_parts( - data=schemas.SessionsSearchPayloadSchema.parse_obj(a["filter"]), - error_status=None, errors_only=False, - favorite_only=False, issue=None, project_id=a["projectId"], - user_id=None) - subQ = f"""SELECT COUNT(session_id) AS value - {query_part}""" - else: - colDef = LeftToDb[a["query"]["left"]] - subQ = f"""SELECT {colDef["formula"]} AS value - FROM {colDef["table"]} - WHERE project_id = %(project_id)s - {"AND " + colDef["condition"] if colDef.get("condition") is not None else ""}""" - j_s = colDef.get("joinSessions", True) - - q = f"""SELECT coalesce(value,0) AS value, coalesce(value,0) {a["query"]["operator"]} {a["query"]["right"]} AS valid""" - - # if len(colDef.group) > 0 { - # subQ = subQ.Column(colDef.group + " AS group_value") - # subQ = subQ.GroupBy(colDef.group) - # q = q.Column("group_value") - # } - - if a["detectionMethod"] == schemas.AlertDetectionMethod.threshold: - if a["seriesId"] is not None: - q += f""" FROM ({subQ}) AS stat""" - else: - q += f""" FROM ({subQ} AND timestamp>=%(startDate)s - {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}) AS stat""" - params = {**params, **full_args, "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000} - else: - if a["options"]["change"] == schemas.AlertDetectionChangeType.change: - # if len(colDef.group) > 0: - # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) - # sub2, args2, _ := subQ.Where( - # sq.And{ - # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # sq.Expr("timestamp>=$4 ", time.Now().Unix()-2 * a.Options.CurrentPeriod * 60), - # }).ToSql() - # sub1 := sq.Select("group_value", "(stat1.value-stat2.value) AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - # q = q.FromSelect(sub1, "stat") - # else: - if a["seriesId"] is not None: - sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") - sub1 = f"SELECT (({subQ})-({sub2})) AS value" - q += f" FROM ( {sub1} ) AS stat" - params = {**params, **full_args, - "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, - "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} - else: - sub1 = f"""{subQ} AND timestamp>=%(startDate)s - {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" - params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 - sub2 = f"""{subQ} AND timestamp<%(startDate)s - AND timestamp>=%(timestamp_sub2)s - {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" - params["timestamp_sub2"] = TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000 - sub1 = f"SELECT (( {sub1} )-( {sub2} )) AS value" - q += f" FROM ( {sub1} ) AS stat" - - else: - # if len(colDef.group) >0 { - # subq1 := subQ.Where(sq.Expr("timestamp>=$2 ", time.Now().Unix()-a.Options.CurrentPeriod * 60)) - # sub2, args2, _ := subQ.Where( - # sq.And{ - # sq.Expr("timestamp<$3 ", time.Now().Unix()-a.Options.CurrentPeriod * 60), - # sq.Expr("timestamp>=$4 ", time.Now().Unix()-a.Options.PreviousPeriod * 60-a.Options.CurrentPeriod * 60), - # }).ToSql() - # sub1 := sq.Select("group_value", "(stat1.value/stat2.value-1)*100 AS value").FromSelect(subq1, "stat1").JoinClause("INNER JOIN ("+sub2+") AS stat2 USING(group_value)", args2...) - # q = q.FromSelect(sub1, "stat") - # } else { - if a["seriesId"] is not None: - sub2 = subQ.replace("%(startDate)s", "%(timestamp_sub2)s").replace("%(endDate)s", "%(startDate)s") - sub1 = f"SELECT (({subQ})/NULLIF(({sub2}),0)-1)*100 AS value" - q += f" FROM ({sub1}) AS stat" - params = {**params, **full_args, - "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, - "timestamp_sub2": TimeUTC.now() \ - - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) \ - * 60 * 1000} - else: - sub1 = f"""{subQ} AND timestamp>=%(startDate)s - {"AND sessions.start_ts >= %(startDate)s" if j_s else ""}""" - params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 - sub2 = f"""{subQ} AND timestamp<%(startDate)s - AND timestamp>=%(timestamp_sub2)s - {"AND sessions.start_ts < %(startDate)s AND sessions.start_ts >= %(timestamp_sub2)s" if j_s else ""}""" - params["timestamp_sub2"] = TimeUTC.now() \ - - (a["options"]["currentPeriod"] + a["options"]["currentPeriod"]) * 60 * 1000 - sub1 = f"SELECT (({sub1})/NULLIF(({sub2}),0)-1)*100 AS value" - q += f" FROM ({sub1}) AS stat" - - return q, params - - -def process(): - notifications = [] - with pg_client.PostgresClient(long_query=True) as cur: - query = """SELECT tenant_id, - alert_id, - project_id, - detection_method, - query, - options, - (EXTRACT(EPOCH FROM alerts.created_at) * 1000)::BIGINT AS created_at, - alerts.name, - alerts.series_id, - filter - FROM public.alerts - LEFT JOIN metric_series USING (series_id) - INNER JOIN projects USING (project_id) - WHERE alerts.deleted_at ISNULL - AND alerts.active - AND projects.active - AND projects.deleted_at ISNULL - AND (alerts.series_id ISNULL OR metric_series.deleted_at ISNULL) - ORDER BY alerts.created_at;""" - cur.execute(query=query) - all_alerts = helper.list_to_camel_case(cur.fetchall()) - - with pg_client.PostgresClient() as cur: - for alert in all_alerts: - if can_check(alert): - print(f"Querying alertId:{alert['alertId']} name: {alert['name']}") - query, params = Build(alert) - query = cur.mogrify(query, params) - # print(alert) - # print(query) - try: - cur.execute(query) - result = cur.fetchone() - if result["valid"]: - print("Valid alert, notifying users") - notifications.append({ - "alertId": alert["alertId"], - "tenantId": alert["tenantId"], - "title": alert["name"], - "description": f"has been triggered, {alert['query']['left']} = {result['value']} ({alert['query']['operator']} {alert['query']['right']}).", - "buttonText": "Check metrics for more details", - "buttonUrl": f"/{alert['projectId']}/metrics", - "imageUrl": None, - "options": {"source": "ALERT", "sourceId": alert["alertId"], - "sourceMeta": alert["detectionMethod"], - "message": alert["options"]["message"], "projectId": alert["projectId"], - "data": {"title": alert["name"], - "limitValue": alert["query"]["right"], "actualValue": result["value"], - "operator": alert["query"]["operator"], - "trigger": alert["query"]["left"], - "alertId": alert["alertId"], - "detectionMethod": alert["detectionMethod"], - "currentPeriod": alert["options"]["currentPeriod"], - "previousPeriod": alert["options"]["previousPeriod"], - "createdAt": TimeUTC.now()}}, - }) - except Exception as e: - print(f"!!!Error while running alert query for alertId:{alert['alertId']}") - print(str(e)) - print(query) - if len(notifications) > 0: - cur.execute( - cur.mogrify(f"""UPDATE public.Alerts - SET options = options||'{{"lastNotification":{TimeUTC.now()}}}'::jsonb - WHERE alert_id IN %(ids)s;""", {"ids": tuple([n["alertId"] for n in notifications])})) - if len(notifications) > 0: - alerts.process_notifications(notifications) From dff5155d39c61230d8ab5610601376ac749597d8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 11 Jan 2022 12:14:25 +0100 Subject: [PATCH 078/206] feat(DB): fixed init DB queries --- ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 2 +- scripts/helm/db/init_dbs/postgresql/init_schema.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 451c6b2a0..527e0f0ae 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -787,7 +787,7 @@ $$ PRIMARY KEY (session_id, message_id) ); CREATE INDEX inputs_session_id_idx ON events.inputs (session_id); - CREATE INDEX inputs_label_idx ON events.inputs (label, value); + CREATE INDEX inputs_label_value_idx ON events.inputs (label, value); CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); CREATE INDEX inputs_label_idx ON events.inputs (label); CREATE INDEX inputs_timestamp_idx ON events.inputs (timestamp); diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 1432af17c..4cd523e2b 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -779,7 +779,7 @@ $$ PRIMARY KEY (session_id, message_id) ); CREATE INDEX inputs_session_id_idx ON events.inputs (session_id); - CREATE INDEX inputs_label_idx ON events.inputs (label, value); + CREATE INDEX inputs_label_value_idx ON events.inputs (label, value); CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); CREATE INDEX inputs_label_idx ON events.inputs (label); CREATE INDEX inputs_timestamp_idx ON events.inputs (timestamp); From ee5d6dca5210ed4b190f23ce06111636b0986271 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 11 Jan 2022 12:20:18 +0100 Subject: [PATCH 079/206] feat(api): fixed index name --- ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 527e0f0ae..182d19a72 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1010,7 +1010,7 @@ $$ }'::jsonb ); CREATE INDEX alerts_project_id_idx ON alerts (project_id); - CREATE INDEX alerts_project_id_idx ON alerts (series_id); + CREATE INDEX alerts_series_id_idx ON alerts (series_id); CREATE TRIGGER on_insert_or_update_or_delete AFTER INSERT OR UPDATE OR DELETE ON alerts From 98b5fad26e02a67d4cc80f9ffc0e89e7e3c4f675 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 11 Jan 2022 12:31:22 +0100 Subject: [PATCH 080/206] feat(api): fixed create DB order --- .../db/init_dbs/postgresql/init_schema.sql | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 4cd523e2b..55d90638e 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -269,33 +269,6 @@ $$ FOR EACH ROW EXECUTE PROCEDURE notify_project(); --- --- alerts.sql --- - - CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); - - CREATE TABLE alerts - ( - alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE, - name text NOT NULL, - description text NULL DEFAULT NULL, - active boolean NOT NULL DEFAULT TRUE, - detection_method alert_detection_method NOT NULL, - query jsonb NOT NULL, - deleted_at timestamp NULL DEFAULT NULL, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - options jsonb NOT NULL DEFAULT '{ - "renotifyInterval": 1440 - }'::jsonb - ); - CREATE INDEX alerts_project_id_idx ON alerts (project_id); - CREATE INDEX alerts_series_id_idx ON alerts (series_id); - CREATE TRIGGER on_insert_or_update_or_delete - AFTER INSERT OR UPDATE OR DELETE - ON alerts - FOR EACH ROW - EXECUTE PROCEDURE notify_alert(); -- --- webhooks.sql --- @@ -966,6 +939,31 @@ $$ CREATE INDEX searches_user_id_is_public_idx ON public.searches (user_id, is_public); CREATE INDEX searches_project_id_idx ON public.searches (project_id); + CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); + + CREATE TABLE alerts + ( + alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE, + name text NOT NULL, + description text NULL DEFAULT NULL, + active boolean NOT NULL DEFAULT TRUE, + detection_method alert_detection_method NOT NULL, + query jsonb NOT NULL, + deleted_at timestamp NULL DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + options jsonb NOT NULL DEFAULT '{ + "renotifyInterval": 1440 + }'::jsonb + ); + CREATE INDEX alerts_project_id_idx ON alerts (project_id); + CREATE INDEX alerts_series_id_idx ON alerts (series_id); + CREATE TRIGGER on_insert_or_update_or_delete + AFTER INSERT OR UPDATE OR DELETE + ON alerts + FOR EACH ROW + EXECUTE PROCEDURE notify_alert(); raise notice 'DB created'; END IF; From 99c31f2090fc9d51a2c215fa40350216ddfcde08 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 11 Jan 2022 18:32:51 +0100 Subject: [PATCH 081/206] feat(api): EE role-projects feat(api): EE project authorizer by user feat(api): EE projects list by user --- ee/api/auth/auth_project.py | 6 +- ee/api/chalicelib/core/projects.py | 61 +++++++++- ee/api/chalicelib/core/roles.py | 112 ++++++++++++------ ee/api/chalicelib/core/users.py | 4 +- ee/api/routers/core_dynamic.py | 5 +- ee/api/routers/ee.py | 4 +- ee/api/schemas_ee.py | 5 + .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 33 ++++-- .../db/init_dbs/postgresql/init_schema.sql | 26 ++-- .../db/init_dbs/postgresql/1.9.9/1.9.9.sql | 18 +-- 10 files changed, 202 insertions(+), 72 deletions(-) diff --git a/ee/api/auth/auth_project.py b/ee/api/auth/auth_project.py index 479681cb8..c1e1d38cd 100644 --- a/ee/api/auth/auth_project.py +++ b/ee/api/auth/auth_project.py @@ -16,10 +16,12 @@ class ProjectAuthorizer: return current_user: schemas.CurrentContext = await OR_context(request) project_identifier = request.path_params[self.project_identifier] + user_id = current_user.user_id if request.state.authorizer_identity == "jwt" else None if (self.project_identifier == "projectId" \ - and not projects.is_authorized(project_id=project_identifier, tenant_id=current_user.tenant_id)) \ + and not projects.is_authorized(project_id=project_identifier, tenant_id=current_user.tenant_id, + user_id=user_id)) \ or (self.project_identifier.lower() == "projectKey" \ and not projects.is_authorized(project_id=projects.get_internal_project_id(project_identifier), - tenant_id=current_user.tenant_id)): + tenant_id=current_user.tenant_id, user_id=user_id)): print("unauthorized project") raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized project.") diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 75efdd452..7cf8e7e2e 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -42,7 +42,7 @@ def __create(tenant_id, name): def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, stack_integrations=False, version=False, - last_tracker_version=None): + last_tracker_version=None, user_id=None): with pg_client.PostgresClient() as cur: tracker_query = "" if last_tracker_version is not None and len(last_tracker_version) > 0: @@ -54,6 +54,15 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st elif version: tracker_query = ",(SELECT tracker_version FROM public.sessions WHERE sessions.project_id = s.project_id ORDER BY start_ts DESC LIMIT 1) AS tracker_version" + role_query = """INNER JOIN LATERAL (SELECT 1 + FROM users + INNER JOIN roles USING (role_id) + LEFT JOIN roles_projects USING (role_id) + WHERE users.user_id = %(user_id)s + AND users.deleted_at ISNULL + AND users.tenant_id = %(tenant_id)s + AND (roles.all_projects OR roles_projects.project_id = s.project_id) + ) AS role_project ON (TRUE)""" cur.execute( cur.mogrify(f"""\ SELECT @@ -64,10 +73,11 @@ def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, st {tracker_query} FROM public.projects AS s {'LEFT JOIN LATERAL (SELECT COUNT(*) AS count FROM public.integrations WHERE s.project_id = integrations.project_id LIMIT 1) AS stack_integrations ON TRUE' if stack_integrations else ''} + {role_query if user_id is not None else ""} WHERE s.tenant_id =%(tenant_id)s AND s.deleted_at IS NULL ORDER BY s.project_id;""", - {"tenant_id": tenant_id}) + {"tenant_id": tenant_id, "user_id": user_id}) ) rows = cur.fetchall() if recording_state: @@ -124,10 +134,34 @@ def get_project(tenant_id, project_id, include_last_session=False, include_gdpr= return helper.dict_to_camel_case(row) -def is_authorized(project_id, tenant_id): +def is_authorized(project_id, tenant_id, user_id=None): if project_id is None or not str(project_id).isdigit(): return False - return get_project(tenant_id=tenant_id, project_id=project_id) is not None + with pg_client.PostgresClient() as cur: + role_query = """INNER JOIN LATERAL (SELECT 1 + FROM users + INNER JOIN roles USING (role_id) + LEFT JOIN roles_projects USING (role_id) + WHERE users.user_id = %(user_id)s + AND users.deleted_at ISNULL + AND users.tenant_id = %(tenant_id)s + AND (roles.all_projects OR roles_projects.project_id = %(project_id)s) + ) AS role_project ON (TRUE)""" + + query = cur.mogrify(f"""\ + SELECT project_id + FROM public.projects AS s + {role_query if user_id is not None else ""} + where s.tenant_id =%(tenant_id)s + AND s.project_id =%(project_id)s + AND s.deleted_at IS NULL + LIMIT 1;""", + {"tenant_id": tenant_id, "project_id": project_id, "user_id": user_id}) + cur.execute( + query=query + ) + row = cur.fetchone() + return row is not None def create(tenant_id, user_id, data: schemas.CreateProjectSchema, skip_authorization=False): @@ -275,3 +309,22 @@ def get_project_by_key(tenant_id, project_key, include_last_session=False, inclu ) row = cur.fetchone() return helper.dict_to_camel_case(row) + + +def is_authorized_batch(project_ids, tenant_id): + if project_ids is None or not len(project_ids): + return False + with pg_client.PostgresClient() as cur: + query = cur.mogrify("""\ + SELECT project_id + FROM public.projects + where tenant_id =%(tenant_id)s + AND project_id IN %(project_ids)s + AND deleted_at IS NULL;""", + {"tenant_id": tenant_id, "project_ids": tuple(project_ids)}) + + cur.execute( + query=query + ) + rows = cur.fetchall() + return [r["project_id"] for r in rows] diff --git a/ee/api/chalicelib/core/roles.py b/ee/api/chalicelib/core/roles.py index 8ba62091a..e8d72d29f 100644 --- a/ee/api/chalicelib/core/roles.py +++ b/ee/api/chalicelib/core/roles.py @@ -1,64 +1,111 @@ -from chalicelib.core import users +import schemas_ee +from chalicelib.core import users, projects from chalicelib.utils import pg_client, helper from chalicelib.utils.TimeUTC import TimeUTC -def update(tenant_id, user_id, role_id, changes): +def update(tenant_id, user_id, role_id, data: schemas_ee.RolePayloadSchema): admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} - if len(changes.keys()) == 0: - return None - ALLOW_EDIT = ["name", "description", "permissions"] - sub_query = [] - for key in changes.keys(): - if key in ALLOW_EDIT: - sub_query.append(f"{helper.key_to_snake_case(key)} = %({key})s") + if not data.all_projects and (data.projects is None or len(data.projects) == 0): + return {"errors": ["must specify a project or all projects"]} + if data.projects is not None and len(data.projects) > 0 and not data.all_projects: + data.projects = projects.is_authorized_batch(project_ids=data.projects, tenant_id=tenant_id) with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify(f"""\ + cur.mogrify("""SELECT 1 + FROM public.roles + WHERE role_id = %(role_id)s + AND tenant_id = %(tenant_id)s + AND protected = TRUE + LIMIT 1;""", + {"tenant_id": tenant_id, "role_id": role_id}) + ) + if cur.fetchone() is not None: + return {"errors": ["this role is protected"]} + cur.execute( + cur.mogrify("""\ UPDATE public.roles - SET {" ,".join(sub_query)} + SET name= %(name)s, + description= %(description)s, + permissions= %(permissions)s, + all_projects= %(all_projects)s WHERE role_id = %(role_id)s AND tenant_id = %(tenant_id)s AND deleted_at ISNULL AND protected = FALSE - RETURNING *;""", - {"tenant_id": tenant_id, "role_id": role_id, **changes}) + RETURNING *, COALESCE((SELECT ARRAY_AGG(project_id) + FROM roles_projects WHERE roles_projects.role_id=%(role_id)s),'{}') AS projects;""", + {"tenant_id": tenant_id, "role_id": role_id, **data.dict()}) ) row = cur.fetchone() row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + if not data.all_projects: + d_projects = [i for i in row["projects"] if i not in data.projects] + if len(d_projects) > 0: + cur.execute( + cur.mogrify( + "DELETE FROM roles_projects WHERE role_id=%(role_id)s AND project_id IN %(project_ids)s", + {"role_id": role_id, "project_ids": tuple(d_projects)}) + ) + n_projects = [i for i in data.projects if i not in row["projects"]] + if len(n_projects) > 0: + cur.execute( + cur.mogrify( + f"""INSERT INTO roles_projects(role_id, project_id) + VALUES {",".join([f"(%(role_id)s,%(project_id_{i})s)" for i in range(len(n_projects))])}""", + {"role_id": role_id, **{f"project_id_{i}": p for i, p in enumerate(n_projects)}}) + ) + row["projects"] = data.projects + return helper.dict_to_camel_case(row) -def create(tenant_id, user_id, name, description, permissions): +def create(tenant_id, user_id, data: schemas_ee.RolePayloadSchema): admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} - + if not data.all_projects and (data.projects is None or len(data.projects) == 0): + return {"errors": ["must specify a project or all projects"]} + if data.projects is not None and len(data.projects) > 0 and not data.all_projects: + data.projects = projects.is_authorized_batch(project_ids=data.projects, tenant_id=tenant_id) with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify("""INSERT INTO roles(tenant_id, name, description, permissions) - VALUES (%(tenant_id)s, %(name)s, %(description)s, %(permissions)s::text[]) + cur.mogrify("""INSERT INTO roles(tenant_id, name, description, permissions, all_projects) + VALUES (%(tenant_id)s, %(name)s, %(description)s, %(permissions)s::text[], %(all_projects)s) RETURNING *;""", - {"tenant_id": tenant_id, "name": name, "description": description, "permissions": permissions}) + {"tenant_id": tenant_id, "name": data.name, "description": data.description, + "permissions": data.permissions, "all_projects": data.all_projects}) ) row = cur.fetchone() row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + if not data.all_projects: + role_id = row["role_id"] + cur.execute( + cur.mogrify(f"""INSERT INTO roles_projects(role_id, project_id) + VALUES {",".join(f"(%(role_id)s,%(project_id_{i})s)" for i in range(len(data.projects)))};""", + {"role_id": role_id, **{f"project_id_{i}": p for i, p in enumerate(data.projects)}}) + ) return helper.dict_to_camel_case(row) def get_roles(tenant_id): with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify("""SELECT * - FROM public.roles - where tenant_id =%(tenant_id)s - AND deleted_at IS NULL - ORDER BY role_id;""", + cur.mogrify("""SELECT *, COALESCE(projects, '{}') AS projects + FROM public.roles + LEFT JOIN LATERAL (SELECT array_agg(project_id) AS projects + FROM roles_projects + INNER JOIN projects USING (project_id) + WHERE roles_projects.role_id = roles.role_id + AND projects.deleted_at ISNULL ) AS role_projects ON (TRUE) + WHERE tenant_id =%(tenant_id)s + AND deleted_at IS NULL + ORDER BY role_id;""", {"tenant_id": tenant_id}) ) rows = cur.fetchall() @@ -71,11 +118,10 @@ def get_role_by_name(tenant_id, name): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify("""SELECT * - FROM public.roles - where tenant_id =%(tenant_id)s - AND deleted_at IS NULL - AND name ILIKE %(name)s - ;""", + FROM public.roles + where tenant_id =%(tenant_id)s + AND deleted_at IS NULL + AND name ILIKE %(name)s;""", {"tenant_id": tenant_id, "name": name}) ) row = cur.fetchone() @@ -92,11 +138,11 @@ def delete(tenant_id, user_id, role_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify("""SELECT 1 - FROM public.roles - WHERE role_id = %(role_id)s - AND tenant_id = %(tenant_id)s - AND protected = TRUE - LIMIT 1;""", + FROM public.roles + WHERE role_id = %(role_id)s + AND tenant_id = %(tenant_id)s + AND protected = TRUE + LIMIT 1;""", {"tenant_id": tenant_id, "role_id": role_id}) ) if cur.fetchone() is not None: diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index 7838a68ac..b1945ac8d 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -482,7 +482,7 @@ def change_password(tenant_id, user_id, email, old_password, new_password): c = tenants.get_by_tenant_id(tenant_id) c.pop("createdAt") c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True, - stack_integrations=True) + stack_integrations=True, user_id=user_id) c["smtp"] = helper.has_smtp() c["iceServers"] = assist.get_ice_servers() return { @@ -510,7 +510,7 @@ def set_password_invitation(tenant_id, user_id, new_password): c = tenants.get_by_tenant_id(tenant_id) c.pop("createdAt") c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True, - stack_integrations=True) + stack_integrations=True, user_id=user_id) c["smtp"] = helper.has_smtp() c["iceServers"] = assist.get_ice_servers() return { diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index bab3689ff..6979eccb3 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -52,7 +52,7 @@ def login(data: schemas.UserLoginSchema = Body(...)): c = tenants.get_by_tenant_id(tenant_id) c.pop("createdAt") c["projects"] = projects.get_projects(tenant_id=tenant_id, recording_state=True, recorded=True, - stack_integrations=True, version=True) + stack_integrations=True, version=True, user_id=r["id"]) c["smtp"] = helper.has_smtp() c["iceServers"] = assist.get_ice_servers() r["smtp"] = c["smtp"] @@ -195,7 +195,8 @@ def search_sessions_by_metadata(key: str, value: str, projectId: Optional[int] = if key is None or value is None or len(value) == 0 and len(key) == 0: return {"errors": ["please provide a key&value for search"]} - if projectId is not None and not projects.is_authorized(project_id=projectId, tenant_id=context.tenant_id): + if projectId is not None and not projects.is_authorized(project_id=projectId, tenant_id=context.tenant_id, + user_id=context.user_id): return {"errors": ["unauthorized project"]} if len(value) == 0: return {"errors": ["please provide a value for search"]} diff --git a/ee/api/routers/ee.py b/ee/api/routers/ee.py index 52ffaad8f..1a9589eaa 100644 --- a/ee/api/routers/ee.py +++ b/ee/api/routers/ee.py @@ -23,7 +23,7 @@ def get_roles(context: schemas.CurrentContext = Depends(OR_context)): @app.post('/client/roles', tags=["client", "roles"]) @app.put('/client/roles', tags=["client", "roles"]) def add_role(data: schemas_ee.RolePayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - data = roles.create(tenant_id=context.tenant_id, user_id=context.user_id, **data.dict()) + data = roles.create(tenant_id=context.tenant_id, user_id=context.user_id, data=data) if "errors" in data: return data @@ -36,7 +36,7 @@ def add_role(data: schemas_ee.RolePayloadSchema = Body(...), context: schemas.Cu @app.put('/client/roles/{roleId}', tags=["client", "roles"]) def edit_role(roleId: int, data: schemas_ee.RolePayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - data = roles.update(tenant_id=context.tenant_id, user_id=context.user_id, role_id=roleId, changes=data.dict()) + data = roles.update(tenant_id=context.tenant_id, user_id=context.user_id, role_id=roleId, data=data) if "errors" in data: return data diff --git a/ee/api/schemas_ee.py b/ee/api/schemas_ee.py index e278f3077..59a58f94b 100644 --- a/ee/api/schemas_ee.py +++ b/ee/api/schemas_ee.py @@ -9,6 +9,11 @@ class RolePayloadSchema(BaseModel): name: str = Field(...) description: Optional[str] = Field(None) permissions: List[str] = Field(...) + all_projects: bool = Field(True) + projects: List[int] = Field([]) + + class Config: + alias_generator = schemas.attribute_to_camel_case class CreateMemberSchema(schemas.CreateMemberSchema): diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 75130e2f4..0bdee3eb8 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -37,9 +37,9 @@ CREATE INDEX IF NOT EXISTS errors_session_id_timestamp_error_id_idx ON events.er CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); CREATE INDEX IF NOT EXISTS errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); -CREATE INDEX ON events.resources (timestamp); -CREATE INDEX ON events.resources (success); -CREATE INDEX ON public.projects (project_key); +CREATE INDEX IF NOT EXISTS resources_timestamp_idx ON events.resources (timestamp); +CREATE INDEX IF NOT EXISTS resources_success_idx ON events.resources (success); +CREATE INDEX IF NOT EXISTS projects_project_key_idx ON public.projects (project_key); CREATE INDEX IF NOT EXISTS resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); @@ -63,9 +63,9 @@ ALTER TABLE tenants ALTER COLUMN name SET NOT NULL; ALTER TABLE sessions - ADD COLUMN utm_source text NULL DEFAULT NULL, - ADD COLUMN utm_medium text NULL DEFAULT NULL, - ADD COLUMN utm_campaign text NULL DEFAULT NULL; + ADD COLUMN IF NOT EXISTS utm_source text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS utm_medium text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS utm_campaign text NULL DEFAULT NULL; CREATE INDEX IF NOT EXISTS sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); @@ -85,7 +85,7 @@ CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.perfor CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; -CREATE TABLE metrics +CREATE TABLE IF NOT EXISTS metrics ( metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -96,7 +96,7 @@ CREATE TABLE metrics deleted_at timestamp ); CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); -CREATE TABLE metric_series +CREATE TABLE IF NOT EXISTS metric_series ( series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, @@ -110,7 +110,7 @@ CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series ( CREATE INDEX IF NOT EXISTS funnels_project_id_idx ON public.funnels (project_id); -CREATE TABLE searches +CREATE TABLE IF NOT EXISTS searches ( search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -127,11 +127,24 @@ CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_i CREATE INDEX IF NOT EXISTS alerts_project_id_idx ON alerts (project_id); ALTER TABLE alerts - ADD COLUMN series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; + ADD COLUMN IF NOT EXISTS series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); UPDATE alerts SET options=jsonb_set(options, '{change}', '"change"') WHERE detection_method = 'change' AND options -> 'change' ISNULL; + +ALTER TABLE roles + ADD COLUMN IF NOT EXISTS all_projects bool NOT NULL DEFAULT TRUE; + +CREATE TABLE IF NOT EXISTS roles_projects +( + role_id integer NOT NULL REFERENCES roles (role_id) ON DELETE CASCADE, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + CONSTRAINT roles_projects_pkey PRIMARY KEY (role_id, project_id) +); +CREATE INDEX IF NOT EXISTS roles_projects_role_id_idx ON roles_projects (role_id); +CREATE INDEX IF NOT EXISTS roles_projects_project_id_idx ON roles_projects (project_id); + COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 182d19a72..e29b8b49b 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -137,14 +137,15 @@ $$ CREATE TABLE roles ( - role_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, - name text NOT NULL, - description text DEFAULT NULL, - permissions text[] NOT NULL DEFAULT '{}', - protected bool NOT NULL DEFAULT FALSE, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - deleted_at timestamp NULL DEFAULT NULL + role_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, + name text NOT NULL, + description text DEFAULT NULL, + permissions text[] NOT NULL DEFAULT '{}', + protected bool NOT NULL DEFAULT FALSE, + all_projects bool NOT NULL DEFAULT TRUE, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + deleted_at timestamp NULL DEFAULT NULL ); CREATE TYPE user_role AS ENUM ('owner', 'admin', 'member'); @@ -293,6 +294,15 @@ $$ FOR EACH ROW EXECUTE PROCEDURE notify_project(); + CREATE TABLE roles_projects + ( + role_id integer NOT NULL REFERENCES roles (role_id) ON DELETE CASCADE, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + CONSTRAINT roles_projects_pkey PRIMARY KEY (role_id, project_id) + ); + CREATE INDEX roles_projects_role_id_idx ON roles_projects (role_id); + CREATE INDEX roles_projects_project_id_idx ON roles_projects (project_id); + -- --- webhooks.sql --- create type webhook_type as enum ('webhook', 'slack', 'email'); diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql index 712990680..7fe9e83e4 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql @@ -9,13 +9,13 @@ CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); -CREATE INDEX ON events.errors (timestamp); -CREATE INDEX ON public.projects (project_key); +CREATE INDEX IF NOT EXISTS errors_timestamp_idx ON events.errors (timestamp); +CREATE INDEX IF NOT EXISTS projects_project_key_idx ON public.projects (project_key); ALTER TABLE sessions - ADD COLUMN utm_source text NULL DEFAULT NULL, - ADD COLUMN utm_medium text NULL DEFAULT NULL, - ADD COLUMN utm_campaign text NULL DEFAULT NULL; + ADD COLUMN IF NOT EXISTS utm_source text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS utm_medium text NULL DEFAULT NULL, + ADD COLUMN IF NOT EXISTS utm_campaign text NULL DEFAULT NULL; CREATE INDEX IF NOT EXISTS sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); @@ -36,7 +36,7 @@ CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.perfor CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; -CREATE TABLE metrics +CREATE TABLE IF NOT EXISTS metrics ( metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -47,7 +47,7 @@ CREATE TABLE metrics deleted_at timestamp ); CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); -CREATE TABLE metric_series +CREATE TABLE IF NOT EXISTS metric_series ( series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, @@ -62,7 +62,7 @@ CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series ( CREATE INDEX IF NOT EXISTS funnels_project_id_idx ON public.funnels (project_id); -CREATE TABLE searches +CREATE TABLE IF NOT EXISTS searches ( search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -79,7 +79,7 @@ CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_i CREATE INDEX IF NOT EXISTS alerts_project_id_idx ON alerts (project_id); ALTER TABLE alerts - ADD COLUMN series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; + ADD COLUMN IF NOT EXISTS series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE; CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); UPDATE alerts From 0f44ae19090530e3a4eae08977f298ce7e477f04 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Jan 2022 11:07:29 +0100 Subject: [PATCH 082/206] feat(alerts): EE changed build script feat(api): EE changed get roles --- api/build_alerts.sh | 2 +- ee/api/chalicelib/core/roles.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/build_alerts.sh b/api/build_alerts.sh index 83fac6f83..51504a276 100644 --- a/api/build_alerts.sh +++ b/api/build_alerts.sh @@ -23,7 +23,7 @@ function make_submodule() { [[ $1 == "ee" ]] && { # -- this part was generated by modules_lister.py -- mkdir alerts - cp -R ./{app_alerts,schemas}.py ./alerts/ + cp -R ./{app_alerts,schemas,schemas_ee}.py ./alerts/ mkdir -p ./alerts/chalicelib/ cp -R ./chalicelib/__init__.py ./alerts/chalicelib/ mkdir -p ./alerts/chalicelib/core/ diff --git a/ee/api/chalicelib/core/roles.py b/ee/api/chalicelib/core/roles.py index e8d72d29f..5bd80dc06 100644 --- a/ee/api/chalicelib/core/roles.py +++ b/ee/api/chalicelib/core/roles.py @@ -96,7 +96,7 @@ def create(tenant_id, user_id, data: schemas_ee.RolePayloadSchema): def get_roles(tenant_id): with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify("""SELECT *, COALESCE(projects, '{}') AS projects + cur.mogrify("""SELECT roles.*, COALESCE(projects, '{}') AS projects FROM public.roles LEFT JOIN LATERAL (SELECT array_agg(project_id) AS projects FROM roles_projects From 172669bc28236b0104a2923b63081c6b25b658fa Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Jan 2022 14:40:50 +0100 Subject: [PATCH 083/206] feat(http): added missing mobile bucket --- scripts/helm/db/bucket_policy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/db/bucket_policy.sh b/scripts/helm/db/bucket_policy.sh index 67e6adb77..65ea068e5 100644 --- a/scripts/helm/db/bucket_policy.sh +++ b/scripts/helm/db/bucket_policy.sh @@ -1,5 +1,5 @@ #!/bin/bash -buckets=("mobs" "sessions-assets" "static" "sourcemaps") +buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets") mc alias set minio http://localhost:9000 $1 $2 From 13a3ec577cec0359aba3eb31d017703c011fc3d6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 12 Jan 2022 14:54:08 +0100 Subject: [PATCH 084/206] feat(DB): changed version number --- .../db/init_dbs/clickhouse/1.4.0/1.4.0.sql | 4 ++++ .../db/init_dbs/clickhouse/1.9.9/1.9.9.sql | 4 ---- .../{1.9.9/1.9.9.sql => 1.4.0/1.4.0.sql} | 19 ++++++++++++++++++- .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 19 ------------------- .../db/init_dbs/postgresql/init_schema.sql | 2 +- .../{1.9.9/1.9.9.sql => 1.4.0/1.4.0.sql} | 2 +- .../db/init_dbs/postgresql/1.5.0/1.5.0.sql | 0 .../db/init_dbs/postgresql/init_schema.sql | 2 +- 8 files changed, 25 insertions(+), 27 deletions(-) create mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.4.0/1.4.0.sql delete mode 100644 ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql rename ee/scripts/helm/db/init_dbs/postgresql/{1.9.9/1.9.9.sql => 1.4.0/1.4.0.sql} (93%) delete mode 100644 ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql rename scripts/helm/db/init_dbs/postgresql/{1.9.9/1.9.9.sql => 1.4.0/1.4.0.sql} (99%) delete mode 100644 scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.4.0/1.4.0.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.4.0/1.4.0.sql new file mode 100644 index 000000000..e259bdf69 --- /dev/null +++ b/ee/scripts/helm/db/init_dbs/clickhouse/1.4.0/1.4.0.sql @@ -0,0 +1,4 @@ +ALTER TABLE sessions + ADD COLUMN IF NOT EXISTS utm_source Nullable(String), + ADD COLUMN IF NOT EXISTS utm_medium Nullable(String), + ADD COLUMN IF NOT EXISTS utm_campaign Nullable(String); diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql deleted file mode 100644 index d427688f5..000000000 --- a/ee/scripts/helm/db/init_dbs/clickhouse/1.9.9/1.9.9.sql +++ /dev/null @@ -1,4 +0,0 @@ -ALTER TABLE sessions - ADD COLUMN utm_source Nullable(String), - ADD COLUMN utm_medium Nullable(String), - ADD COLUMN utm_campaign Nullable(String); diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql similarity index 93% rename from ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql rename to ee/scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql index 0bdee3eb8..94d5fced6 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql @@ -2,9 +2,26 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.9.9-ee' +SELECT 'v1.4.0-ee' $$ LANGUAGE sql IMMUTABLE; +CREATE TABLE IF NOT EXISTS traces +( + user_id integer NULL REFERENCES users (user_id) ON DELETE CASCADE, + tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, + created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, + auth text NULL, + action text NOT NULL, + method text NOT NULL, + path_format text NOT NULL, + endpoint text NOT NULL, + payload jsonb NULL, + parameters jsonb NULL, + status int NULL +); +CREATE INDEX IF NOT EXISTS traces_user_id_idx ON traces (user_id); +CREATE INDEX IF NOT EXISTS traces_tenant_id_idx ON traces (tenant_id); + CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); CREATE INDEX IF NOT EXISTS pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; diff --git a/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql deleted file mode 100644 index 8b9fe6cfd..000000000 --- a/ee/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql +++ /dev/null @@ -1,19 +0,0 @@ -BEGIN; -CREATE TABLE traces -( - user_id integer NULL REFERENCES users (user_id) ON DELETE CASCADE, - tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, - created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, - auth text NULL, - action text NOT NULL, - method text NOT NULL, - path_format text NOT NULL, - endpoint text NOT NULL, - payload jsonb NULL, - parameters jsonb NULL, - status int NULL -); -CREATE INDEX traces_user_id_idx ON traces (user_id); -CREATE INDEX traces_tenant_id_idx ON traces (tenant_id); - -COMMIT; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index e29b8b49b..35f3a5dd4 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS events; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.9.9-ee' +SELECT 'v1.4.0-ee' $$ LANGUAGE sql IMMUTABLE; -- --- accounts.sql --- diff --git a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql b/scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql similarity index 99% rename from scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql rename to scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql index 7fe9e83e4..2f021d1ee 100644 --- a/scripts/helm/db/init_dbs/postgresql/1.9.9/1.9.9.sql +++ b/scripts/helm/db/init_dbs/postgresql/1.4.0/1.4.0.sql @@ -2,7 +2,7 @@ BEGIN; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.9.9' +SELECT 'v1.4.0' $$ LANGUAGE sql IMMUTABLE; CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); diff --git a/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql b/scripts/helm/db/init_dbs/postgresql/1.5.0/1.5.0.sql deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 55d90638e..4607c2759 100644 --- a/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -6,7 +6,7 @@ CREATE SCHEMA IF NOT EXISTS events; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS $$ -SELECT 'v1.9.9' +SELECT 'v1.4.0' $$ LANGUAGE sql IMMUTABLE; -- --- accounts.sql --- From 8bd852d79cde41bcec3f63c6f588313309461862 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 14:25:06 +0100 Subject: [PATCH 085/206] feat(api): fixed update alerts --- api/chalicelib/core/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index ef35d48b3..42411b8bc 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -110,7 +110,7 @@ def update(id, data: schemas.AlertSchema): UPDATE public.alerts SET name = %(name)s, description = %(description)s, - active = %(active)s, + active = TRUE, detection_method = %(detection_method)s, query = %(query)s, options = %(options)s, From 1d2b8481cde9b844ffed85cf0d78aebb8a5f5a02 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 14:44:51 +0100 Subject: [PATCH 086/206] feat(api): fixed update alerts --- api/chalicelib/core/alerts.py | 43 +---------------------------------- api/schemas.py | 2 +- 2 files changed, 2 insertions(+), 43 deletions(-) diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index 42411b8bc..3bb632c30 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -7,9 +7,6 @@ from chalicelib.utils import pg_client, helper, email_helper from chalicelib.utils.TimeUTC import TimeUTC -# ALLOW_UPDATE = ["name", "description", "active", "detectionMethod", "query", "options"] - - def get(id): with pg_client.PostgresClient() as cur: cur.execute( @@ -38,34 +35,6 @@ def get_all(project_id): return all -# SUPPORTED_THRESHOLD = [15, 30, 60, 120, 240, 1440] -# -# -# def __transform_structure(data): -# if data.get("options") is None: -# return f"Missing 'options'", None -# if data["options"].get("currentPeriod") not in SUPPORTED_THRESHOLD: -# return f"Unsupported currentPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None -# if data["options"].get("previousPeriod", 15) not in SUPPORTED_THRESHOLD: -# return f"Unsupported previousPeriod, please provide one of these values {SUPPORTED_THRESHOLD}", None -# if data["options"].get("renotifyInterval") is None: -# data["options"]["renotifyInterval"] = 720 -# data["query"]["right"] = float(data["query"]["right"]) -# data["query"] = json.dumps(data["query"]) -# data["description"] = data["description"] if data.get("description") is not None and len( -# data["description"]) > 0 else None -# if data.get("options"): -# messages = [] -# for m in data["options"].get("message", []): -# if m.get("value") is None: -# continue -# m["value"] = str(m["value"]) -# messages.append(m) -# data["options"]["message"] = messages -# data["options"] = json.dumps(data["options"]) -# return None, data - - def __process_circular(alert): if alert is None: return None @@ -83,7 +52,7 @@ def create(project_id, data: schemas.AlertSchema): cur.execute( cur.mogrify("""\ INSERT INTO public.alerts(project_id, name, description, detection_method, query, options, series_id) - VALUES (%(project_id)s, %(name)s, %(description)s, %(detectionMethod)s, %(query)s, %(options)s::jsonb, %(series_id)s) + VALUES (%(project_id)s, %(name)s, %(description)s, %(detection_method)s, %(query)s, %(options)s::jsonb, %(series_id)s) RETURNING *;""", {"project_id": project_id, **data}) ) @@ -96,15 +65,6 @@ def update(id, data: schemas.AlertSchema): data["query"] = json.dumps(data["query"]) data["options"] = json.dumps(data["options"]) - # changes = {k: changes[k] for k in changes.keys() if k in ALLOW_UPDATE} - # err, changes = __transform_structure(changes) - # if err is not None: - # return {"errors": [err]} - # updateq = [] - # for k in changes.keys(): - # updateq.append(f"{helper.key_to_snake_case(k)} = %({k})s") - # if len(updateq) == 0: - # return {"errors": ["nothing to update"]} with pg_client.PostgresClient() as cur: query = cur.mogrify("""\ UPDATE public.alerts @@ -144,7 +104,6 @@ def process_notifications(data): BATCH_SIZE = 200 for t in full.keys(): for i in range(0, len(full[t]), BATCH_SIZE): - # helper.async_post(config('alert_ntf') % t, {"notifications": full[t][i:i + BATCH_SIZE]}) notifications_list = full[t][i:i + BATCH_SIZE] if t == "slack": diff --git a/api/schemas.py b/api/schemas.py index 6a33c20a4..0dfd949ac 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -325,7 +325,7 @@ class AlertDetectionMethod(str, Enum): class AlertSchema(BaseModel): name: str = Field(...) - detectionMethod: AlertDetectionMethod = Field(...) + detection_method: AlertDetectionMethod = Field(...) description: Optional[str] = Field(None) options: _AlertOptionSchema = Field(...) query: _AlertQuerySchema = Field(...) From aad6bceca67897f625146d6e1dcd13549808204b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 15:30:54 +0100 Subject: [PATCH 087/206] feat(api): updated HTML date --- api/chalicelib/utils/html/alert_notification.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/chalicelib/utils/html/alert_notification.html b/api/chalicelib/utils/html/alert_notification.html index 881d6ffb0..2d63341f3 100644 --- a/api/chalicelib/utils/html/alert_notification.html +++ b/api/chalicelib/utils/html/alert_notification.html @@ -38,7 +38,7 @@

- Sent with ♡ from OpenReplay © 2021 - All rights reserved.

+ Sent with ♡ from OpenReplay © 2022 - All rights reserved.

https://openreplay.com/

From 026753208102fd6b83f19f32d7c4833a66cd266d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 16:29:36 +0100 Subject: [PATCH 088/206] feat(api): use logger feat(alerts): use logger --- api/app.py | 9 +++++++-- api/app_alerts.py | 2 +- api/chalicelib/core/alerts.py | 14 +++++++------- api/chalicelib/core/alerts_processor.py | 18 ++++++++++-------- api/chalicelib/core/webhook.py | 20 ++++++++------------ ee/api/app.py | 6 +++++- 6 files changed, 38 insertions(+), 31 deletions(-) diff --git a/api/app.py b/api/app.py index 47b200aef..0d4176330 100644 --- a/api/app.py +++ b/api/app.py @@ -1,3 +1,5 @@ +import logging + from apscheduler.schedulers.asyncio import AsyncIOScheduler from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware @@ -60,5 +62,8 @@ Schedule.start() for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs: Schedule.add_job(id=job["func"].__name__, **job) -# for job in Schedule.get_jobs(): -# print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) +for job in Schedule.get_jobs(): + print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + +logging.basicConfig(level=logging.INFO) +logging.getLogger('apscheduler').setLevel(logging.INFO) diff --git a/api/app_alerts.py b/api/app_alerts.py index 60e1b7e48..bdd4702e2 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -24,5 +24,5 @@ app.schedule.add_job(id="alerts_processor", **{"func": alerts_processor.process, for job in app.schedule.get_jobs(): print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) -logging.basicConfig() +logging.basicConfig(level=logging.INFO) logging.getLogger('apscheduler').setLevel(logging.INFO) diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index 3bb632c30..c701f0ce0 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -5,7 +5,7 @@ import schemas from chalicelib.core import notifications, slack, webhook from chalicelib.utils import pg_client, helper, email_helper from chalicelib.utils.TimeUTC import TimeUTC - +import logging def get(id): with pg_client.PostgresClient() as cur: @@ -110,20 +110,20 @@ def process_notifications(data): try: slack.send_batch(notifications_list=notifications_list) except Exception as e: - print("!!!Error while sending slack notifications batch") - print(str(e)) + logging.error("!!!Error while sending slack notifications batch") + logging.error(str(e)) elif t == "email": try: send_by_email_batch(notifications_list=notifications_list) except Exception as e: - print("!!!Error while sending email notifications batch") - print(str(e)) + logging.error("!!!Error while sending email notifications batch") + logging.error(str(e)) elif t == "webhook": try: webhook.trigger_batch(data_list=notifications_list) except Exception as e: - print("!!!Error while sending webhook notifications batch") - print(str(e)) + logging.error("!!!Error while sending webhook notifications batch") + logging.error(str(e)) def send_by_email(notification, destination): diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 43f57a96b..36ea9c501 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -1,3 +1,5 @@ +import logging + import schemas from chalicelib.core import alerts_listener from chalicelib.core import sessions, alerts @@ -80,7 +82,7 @@ def can_check(a) -> bool: else a["options"]["previousPeriod"] if TimeInterval.get(repetitionBase) is None: - print(f"repetitionBase: {repetitionBase} NOT FOUND") + logging.error(f"repetitionBase: {repetitionBase} NOT FOUND") return False return (a["options"]["renotifyInterval"] <= 0 or @@ -200,16 +202,16 @@ def process(): with pg_client.PostgresClient() as cur: for alert in all_alerts: if can_check(alert): - print(f"Querying alertId:{alert['alertId']} name: {alert['name']}") + logging.info(f"Querying alertId:{alert['alertId']} name: {alert['name']}") query, params = Build(alert) query = cur.mogrify(query, params) - # print(alert) - # print(query) + logging.debug(alert) + logging.debug(query) try: cur.execute(query) result = cur.fetchone() if result["valid"]: - print("Valid alert, notifying users") + logging.info("Valid alert, notifying users") notifications.append({ "alertId": alert["alertId"], "tenantId": alert["tenantId"], @@ -232,9 +234,9 @@ def process(): "createdAt": TimeUTC.now()}}, }) except Exception as e: - print(f"!!!Error while running alert query for alertId:{alert['alertId']}") - print(str(e)) - print(query) + logging.error(f"!!!Error while running alert query for alertId:{alert['alertId']}") + logging.error(str(e)) + logging.error(query) if len(notifications) > 0: cur.execute( cur.mogrify(f"""UPDATE public.Alerts diff --git a/api/chalicelib/core/webhook.py b/api/chalicelib/core/webhook.py index 254a46c97..d0b3e2adc 100644 --- a/api/chalicelib/core/webhook.py +++ b/api/chalicelib/core/webhook.py @@ -1,3 +1,5 @@ +import logging + import requests from chalicelib.utils import pg_client, helper @@ -77,12 +79,6 @@ def update(tenant_id, webhook_id, changes, replace_none=False): allow_update = ["name", "index", "authHeader", "endpoint"] with pg_client.PostgresClient() as cur: sub_query = [f"{helper.key_to_snake_case(k)} = %({k})s" for k in changes.keys() if k in allow_update] - print(cur.mogrify(f"""\ - UPDATE public.webhooks - SET {','.join(sub_query)} - WHERE webhook_id =%(id)s AND deleted_at ISNULL - RETURNING webhook_id AS integration_id, webhook_id AS id,*;""", - {"id": webhook_id, **changes})) cur.execute( cur.mogrify(f"""\ UPDATE public.webhooks @@ -152,7 +148,7 @@ def trigger_batch(data_list): if w["destination"] not in webhooks_map: webhooks_map[w["destination"]] = get_by_id(webhook_id=w["destination"]) if webhooks_map[w["destination"]] is None: - print(f"!!Error webhook not found: webhook_id={w['destination']}") + logging.error(f"!!Error webhook not found: webhook_id={w['destination']}") else: __trigger(hook=webhooks_map[w["destination"]], data=w["data"]) @@ -165,10 +161,10 @@ def __trigger(hook, data): r = requests.post(url=hook["endpoint"], json=data, headers=headers) if r.status_code != 200: - print("=======> webhook: something went wrong") - print(r) - print(r.status_code) - print(r.text) + logging.error("=======> webhook: something went wrong") + logging.error(r) + logging.error(r.status_code) + logging.error(r.text) return response = None try: @@ -177,5 +173,5 @@ def __trigger(hook, data): try: response = r.text except: - print("no response found") + logging.info("no response found") return response diff --git a/ee/api/app.py b/ee/api/app.py index 1c731c3f7..15bcc7ac5 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -1,3 +1,4 @@ +import logging import queue from apscheduler.schedulers.asyncio import AsyncIOScheduler @@ -75,7 +76,10 @@ for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs: app.schedule.add_job(id=job["func"].__name__, **job) from chalicelib.core import traces -app.schedule.add_job(id="trace_worker",**traces.cron_jobs[0]) +app.schedule.add_job(id="trace_worker", **traces.cron_jobs[0]) for job in app.schedule.get_jobs(): print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) + +logging.basicConfig(level=logging.INFO) +logging.getLogger('apscheduler').setLevel(logging.INFO) From 49a240612cded2915c8416ea809bb64b7724aed6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 17:13:50 +0100 Subject: [PATCH 089/206] feat(alerts): fix webhook json-serialization issue --- api/app.py | 5 +++-- api/app_alerts.py | 5 ++--- api/chalicelib/core/alerts_processor.py | 6 +++++- ee/api/app.py | 5 +++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/api/app.py b/api/app.py index 0d4176330..d261dadac 100644 --- a/api/app.py +++ b/api/app.py @@ -1,6 +1,7 @@ import logging from apscheduler.schedulers.asyncio import AsyncIOScheduler +from decouple import config from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse @@ -65,5 +66,5 @@ for job in core_crons.cron_jobs + core_dynamic_crons.cron_jobs: for job in Schedule.get_jobs(): print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) -logging.basicConfig(level=logging.INFO) -logging.getLogger('apscheduler').setLevel(logging.INFO) +logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) +logging.getLogger('apscheduler').setLevel(config("LOGLEVEL", default=logging.INFO)) diff --git a/api/app_alerts.py b/api/app_alerts.py index bdd4702e2..57bfcd55d 100644 --- a/api/app_alerts.py +++ b/api/app_alerts.py @@ -20,9 +20,8 @@ app.schedule.start() app.schedule.add_job(id="alerts_processor", **{"func": alerts_processor.process, "trigger": "interval", "minutes": config("ALERTS_INTERVAL", cast=int, default=5), "misfire_grace_time": 20}) - for job in app.schedule.get_jobs(): print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) -logging.basicConfig(level=logging.INFO) -logging.getLogger('apscheduler').setLevel(logging.INFO) +logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) +logging.getLogger('apscheduler').setLevel(config("LOGLEVEL", default=logging.INFO)) diff --git a/api/chalicelib/core/alerts_processor.py b/api/chalicelib/core/alerts_processor.py index 36ea9c501..80973fadd 100644 --- a/api/chalicelib/core/alerts_processor.py +++ b/api/chalicelib/core/alerts_processor.py @@ -1,3 +1,4 @@ +import decimal import logging import schemas @@ -224,7 +225,10 @@ def process(): "sourceMeta": alert["detectionMethod"], "message": alert["options"]["message"], "projectId": alert["projectId"], "data": {"title": alert["name"], - "limitValue": alert["query"]["right"], "actualValue": result["value"], + "limitValue": alert["query"]["right"], + "actualValue": float(result["value"]) \ + if isinstance(result["value"], decimal.Decimal) \ + else result["value"], "operator": alert["query"]["operator"], "trigger": alert["query"]["left"], "alertId": alert["alertId"], diff --git a/ee/api/app.py b/ee/api/app.py index 15bcc7ac5..fdf7f60b8 100644 --- a/ee/api/app.py +++ b/ee/api/app.py @@ -2,6 +2,7 @@ import logging import queue from apscheduler.schedulers.asyncio import AsyncIOScheduler +from decouple import config from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from starlette import status @@ -81,5 +82,5 @@ app.schedule.add_job(id="trace_worker", **traces.cron_jobs[0]) for job in app.schedule.get_jobs(): print({"Name": str(job.id), "Run Frequency": str(job.trigger), "Next Run": str(job.next_run_time)}) -logging.basicConfig(level=logging.INFO) -logging.getLogger('apscheduler').setLevel(logging.INFO) +logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) +logging.getLogger('apscheduler').setLevel(config("LOGLEVEL", default=logging.INFO)) From e9b0809d32fea1079406d948cfe6a2f146b87f49 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Thu, 13 Jan 2022 16:31:26 +0530 Subject: [PATCH 090/206] chore(upgrade): inject ee image Signed-off-by: Rajesh Rajendran --- scripts/helm/upgrade.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helm/upgrade.sh b/scripts/helm/upgrade.sh index f57d2fb07..3c437db9a 100644 --- a/scripts/helm/upgrade.sh +++ b/scripts/helm/upgrade.sh @@ -81,6 +81,7 @@ patch installation_type=1 if [[ ${ENTERPRISE} -eq 1 ]]; then cp -rf ../../ee/scripts/* ../../scripts/ + sed 's/\(image_tag.*[0-9]\)\(-pr\)\?"$/\1\2-ee"/' vars.yaml echo -e "Migrating clickhouse" migration clickhouse fi From 6d0d96df6d5ad7f364f376247fe7c24d972417bc Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 20:35:09 +0100 Subject: [PATCH 091/206] feat(api): fix get projects by role --- api/chalicelib/core/projects.py | 2 +- api/routers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index 2536ee956..8126fccfc 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -43,7 +43,7 @@ def __create(tenant_id, name): @dev.timed def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, stack_integrations=False, version=False, - last_tracker_version=None): + last_tracker_version=None, user_id=None): with pg_client.PostgresClient() as cur: tracker_query = "" if last_tracker_version is not None and len(last_tracker_version) > 0: diff --git a/api/routers/core.py b/api/routers/core.py index 78170298a..dfb411813 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -843,7 +843,7 @@ def signup_handler(data: schemas.UserSignupSchema = Body(...)): def get_projects(last_tracker_version: Optional[str] = None, context: schemas.CurrentContext = Depends(OR_context)): return {"data": projects.get_projects(tenant_id=context.tenant_id, recording_state=True, gdpr=True, recorded=True, stack_integrations=True, version=True, - last_tracker_version=last_tracker_version)} + last_tracker_version=last_tracker_version, user_id=context.user_id)} @app.post('/projects', tags=['projects']) From fdeb6b09fb5a1dbeacc8aaa23a084ce748799039 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 20:52:23 +0100 Subject: [PATCH 092/206] feat(api): EE small change to trigger re-build --- ee/api/chalicelib/core/projects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 7cf8e7e2e..4fce9d9e6 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -187,8 +187,7 @@ def delete(tenant_id, user_id, project_id): return {"errors": ["unauthorized"]} with pg_client.PostgresClient() as cur: cur.execute( - cur.mogrify("""\ - UPDATE public.projects + cur.mogrify("""UPDATE public.projects SET deleted_at = timezone('utc'::text, now()), active = FALSE From 9308d04f927e16df224cb02fb4bc08b78046e9fd Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Thu, 13 Jan 2022 21:12:01 +0100 Subject: [PATCH 093/206] feat(api): EE changed endpoints for role-project --- api/chalicelib/core/projects.py | 2 +- api/routers/core.py | 21 +-------------------- api/routers/core_dynamic.py | 19 +++++++++++++++++++ ee/api/routers/core_dynamic.py | 19 +++++++++++++++++++ 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/api/chalicelib/core/projects.py b/api/chalicelib/core/projects.py index 8126fccfc..2536ee956 100644 --- a/api/chalicelib/core/projects.py +++ b/api/chalicelib/core/projects.py @@ -43,7 +43,7 @@ def __create(tenant_id, name): @dev.timed def get_projects(tenant_id, recording_state=False, gdpr=None, recorded=False, stack_integrations=False, version=False, - last_tracker_version=None, user_id=None): + last_tracker_version=None): with pg_client.PostgresClient() as cur: tracker_query = "" if last_tracker_version is not None and len(last_tracker_version) > 0: diff --git a/api/routers/core.py b/api/routers/core.py index dfb411813..792e4c739 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1,4 +1,4 @@ -from typing import Union, Optional +from typing import Union from decouple import config from fastapi import Depends, Body @@ -839,13 +839,6 @@ def signup_handler(data: schemas.UserSignupSchema = Body(...)): return signup.create_step1(data) -@app.get('/projects', tags=['projects']) -def get_projects(last_tracker_version: Optional[str] = None, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": projects.get_projects(tenant_id=context.tenant_id, recording_state=True, gdpr=True, recorded=True, - stack_integrations=True, version=True, - last_tracker_version=last_tracker_version, user_id=context.user_id)} - - @app.post('/projects', tags=['projects']) @app.put('/projects', tags=['projects']) def create_project(data: schemas.CreateProjectSchema = Body(...), @@ -865,18 +858,6 @@ def delete_project(projectId, context: schemas.CurrentContext = Depends(OR_conte return projects.delete(tenant_id=context.tenant_id, user_id=context.user_id, project_id=projectId) -@app.get('/client', tags=['projects']) -def get_client(context: schemas.CurrentContext = Depends(OR_context)): - r = tenants.get_by_tenant_id(context.tenant_id) - if r is not None: - r.pop("createdAt") - r["projects"] = projects.get_projects(tenant_id=context.tenant_id, recording_state=True, recorded=True, - stack_integrations=True, version=True) - return { - 'data': r - } - - @app.get('/client/new_api_key', tags=['client']) def generate_new_tenant_token(context: schemas.CurrentContext = Depends(OR_context)): return { diff --git a/api/routers/core_dynamic.py b/api/routers/core_dynamic.py index d61d2ebfb..c149266b5 100644 --- a/api/routers/core_dynamic.py +++ b/api/routers/core_dynamic.py @@ -212,3 +212,22 @@ def get_current_plan(context: schemas.CurrentContext = Depends(OR_context)): @public_app.get('/general_stats', tags=["private"], include_in_schema=False) def get_general_stats(): return {"data": {"sessions:": sessions.count_all()}} + + +@app.get('/client', tags=['projects']) +def get_client(context: schemas.CurrentContext = Depends(OR_context)): + r = tenants.get_by_tenant_id(context.tenant_id) + if r is not None: + r.pop("createdAt") + r["projects"] = projects.get_projects(tenant_id=context.tenant_id, recording_state=True, recorded=True, + stack_integrations=True, version=True) + return { + 'data': r + } + + +@app.get('/projects', tags=['projects']) +def get_projects(last_tracker_version: Optional[str] = None, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": projects.get_projects(tenant_id=context.tenant_id, recording_state=True, gdpr=True, recorded=True, + stack_integrations=True, version=True, + last_tracker_version=last_tracker_version)} diff --git a/ee/api/routers/core_dynamic.py b/ee/api/routers/core_dynamic.py index 6979eccb3..cf5e7378f 100644 --- a/ee/api/routers/core_dynamic.py +++ b/ee/api/routers/core_dynamic.py @@ -217,3 +217,22 @@ def get_current_plan(context: schemas.CurrentContext = Depends(OR_context)): @public_app.get('/general_stats', tags=["private"], include_in_schema=False) def get_general_stats(): return {"data": {"sessions:": sessions.count_all()}} + + +@app.get('/client', tags=['projects']) +def get_client(context: schemas.CurrentContext = Depends(OR_context)): + r = tenants.get_by_tenant_id(context.tenant_id) + if r is not None: + r.pop("createdAt") + r["projects"] = projects.get_projects(tenant_id=context.tenant_id, recording_state=True, recorded=True, + stack_integrations=True, version=True, user_id=context.user_id) + return { + 'data': r + } + + +@app.get('/projects', tags=['projects']) +def get_projects(last_tracker_version: Optional[str] = None, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": projects.get_projects(tenant_id=context.tenant_id, recording_state=True, gdpr=True, recorded=True, + stack_integrations=True, version=True, + last_tracker_version=last_tracker_version, user_id=context.user_id)} From cf3efa62e82870a7ec942ca63d9dc16a5743fd43 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 14 Jan 2022 06:47:12 +0530 Subject: [PATCH 094/206] chore(install): check for -pr in releases Signed-off-by: Rajesh Rajendran --- scripts/helm/kube-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/kube-install.sh b/scripts/helm/kube-install.sh index 0a42416bf..056a45de4 100755 --- a/scripts/helm/kube-install.sh +++ b/scripts/helm/kube-install.sh @@ -147,7 +147,7 @@ function enterprise(){ enterprise=1 sed -i "s#enterprise_edition_license.*#enterprise_edition_license: \"${1}\"#g" vars.yaml # Updating image version to be ee - sed -i "s/\(image_tag.*[0-9]\)\"$/\1-ee\"/" vars.yaml + sed 's/\(image_tag.*[0-9]\)\(-pr\)\?"$/\1\2-ee"/' vars.yaml echo "Importing enterprise code..." cp -rf ../../ee/scripts/* ../ } From 6c6a920bcf162a4fe3c3db3f0c4ee6bcc511669c Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 14 Jan 2022 11:36:35 +0100 Subject: [PATCH 095/206] feat(api): EE create project restriction --- ee/api/chalicelib/core/projects.py | 2 ++ ee/api/chalicelib/core/users.py | 1 + 2 files changed, 3 insertions(+) diff --git a/ee/api/chalicelib/core/projects.py b/ee/api/chalicelib/core/projects.py index 4fce9d9e6..2728e5077 100644 --- a/ee/api/chalicelib/core/projects.py +++ b/ee/api/chalicelib/core/projects.py @@ -169,6 +169,8 @@ def create(tenant_id, user_id, data: schemas.CreateProjectSchema, skip_authoriza admin = users.get(user_id=user_id, tenant_id=tenant_id) if not admin["admin"] and not admin["superAdmin"]: return {"errors": ["unauthorized"]} + if admin["roleId"] is not None and not admin["allProjects"]: + return {"errors": ["unauthorized: you need allProjects permission to create a new project"]} return {"data": __create(tenant_id=tenant_id, name=data.name)} diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index b1945ac8d..8f1ef7955 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -274,6 +274,7 @@ def get(user_id, tenant_id): role_id, roles.name AS role_name, roles.permissions, + roles.all_projects, basic_authentication.password IS NOT NULL AS has_password FROM public.users LEFT JOIN public.basic_authentication ON users.user_id=basic_authentication.user_id LEFT JOIN public.roles USING (role_id) From eabc508e8b8304b231f663dc41f41ccdf38fb0c0 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 14 Jan 2022 14:06:16 +0000 Subject: [PATCH 096/206] Helm chart for openreplay installation. (#270) * single db chart * chore(values): update subchart information * chore(helm): global override file * chore(helm): app chart * chore(helm): database-migrate script * chore(helm): pass fromVersion as command * chore(dbMigrate): modularize the job * chore(helm): upgrade hook * chore(helm): upgrade script * chore(helm): db init scripts * fix(uprade): version check * chore(helm): Adding ee charts * fix(helm): clickhouse upgrade script * chore(helm): add enterprise copy support * chore(helm): enterprise migration * fix(helm): clickhouse port * fix(helm): inject env vars * chore(clickhouse): ignore db errors * chore(helm): kafka db upgrade * fix(migration): kafka migration * chore(helm): helm chart to install openreplay * chore(helm): sample override value * chore(helm): cloning main branch * chore(helm): separate chart.yaml for all helm charts. * chore(helm): update version * fix(helm): templates * fix(helm): image name * chore(helm): pinning http port for http service * docs(helm): example for overriding values. * chore(dock): update the comments * removing duplicate helm chart * docs(values): comment for nginx redirect * chore(helm): Adding nginx-ingress commit * chore(helm): Adding default nginx ssl * chore(helm): change nginx image to openresty Ref: https://serverfault.com/questions/638822/nginx-resolver-address-from-etc-resolv-conf * chore(helm): consistent service name * chore(helm): change nginx healthcheck url * chore(helm): init minio * chore(helm): fix password for minio * chore(helm): minio init change workdir to /tmp * chore(helm): ignore error on repeated bucket create * chore(helm): fix enterprise check * fix(helm): storage s3 region * chore(helm): default false for cache_assets * chore(helm): set nginx app version * chore(helm): inject ee image * chore(helm): renaming values.yaml to override.yaml * chore(readme): update readme * chore(helm): renaming vars.yaml * chore(helm): git migrate current version of the app * chore(helm): managing dependencies * fix(helm): clone proper openreplay code * chore(helm): update variable name * chore(install): Adding web install script * chore(install): ignoring apt error messages. * chore(install): change only pre-populated passwords * chore(install): better error messages --- .../databases/charts/clickhouse/.helmignore | 23 + .../databases/charts/clickhouse/Chart.yaml | 23 + .../charts/clickhouse/templates/_helpers.tpl | 62 + .../charts/clickhouse/templates/service.yaml | 19 + .../clickhouse/templates/serviceaccount.yaml | 12 + .../clickhouse/templates/statefulset.yaml | 69 ++ .../databases/charts/clickhouse/values.yaml | 62 + .../helm/databases/charts/kafka/.helmignore | 21 + .../helm/databases/charts/kafka/Chart.yaml | 22 + .../helm/databases/charts/kafka/README.md | 737 ++++++++++++ .../charts/kafka/charts/zookeeper/.helmignore | 21 + .../charts/kafka/charts/zookeeper/Chart.yaml | 20 + .../charts/kafka/charts/zookeeper/README.md | 297 +++++ .../charts/zookeeper/templates/NOTES.txt | 57 + .../charts/zookeeper/templates/_helpers.tpl | 212 ++++ .../charts/zookeeper/templates/configmap.yaml | 17 + .../zookeeper/templates/metrics-svc.yaml | 29 + .../zookeeper/templates/networkpolicy.yaml | 43 + .../templates/poddisruptionbudget.yaml | 21 + .../zookeeper/templates/prometheusrules.yaml | 27 + .../charts/zookeeper/templates/secrets.yaml | 18 + .../zookeeper/templates/serviceaccount.yaml | 15 + .../zookeeper/templates/servicemonitor.yaml | 38 + .../zookeeper/templates/statefulset.yaml | 334 ++++++ .../zookeeper/templates/svc-headless.yaml | 42 + .../kafka/charts/zookeeper/templates/svc.yaml | 40 + .../charts/zookeeper/values-production.yaml | 430 +++++++ .../charts/kafka/charts/zookeeper/values.yaml | 430 +++++++ .../charts/kafka/files/jks/README.md | 10 + .../helm/databases/charts/kafka/kafka.yaml | 521 +++++++++ .../databases/charts/kafka/requirements.lock | 6 + .../databases/charts/kafka/requirements.yaml | 5 + .../charts/kafka/templates/NOTES.txt | 181 +++ .../charts/kafka/templates/_helpers.tpl | 556 +++++++++ .../charts/kafka/templates/configmap.yaml | 16 + .../charts/kafka/templates/extra-list.yaml | 5 + .../charts/kafka/templates/jaas-secret.yaml | 32 + .../charts/kafka/templates/jks-secret.yaml | 19 + .../charts/kafka/templates/jmx-configmap.yaml | 59 + .../kafka/templates/jmx-metrics-svc.yaml | 45 + .../templates/kafka-metrics-deployment.yaml | 87 ++ .../kafka/templates/kafka-metrics-svc.yaml | 45 + .../kafka/templates/log4j-configmap.yaml | 16 + .../kafka/templates/poddisruptionbudget.yaml | 25 + .../charts/kafka/templates/role.yaml | 23 + .../charts/kafka/templates/rolebinding.yaml | 22 + .../kafka/templates/scripts-configmap.yaml | 118 ++ .../kafka/templates/serviceaccount.yaml | 14 + .../templates/servicemonitor-jmx-metrics.yaml | 36 + .../templates/servicemonitor-metrics.yaml | 36 + .../charts/kafka/templates/statefulset.yaml | 435 +++++++ .../kafka/templates/svc-external-access.yaml | 52 + .../charts/kafka/templates/svc-headless.yaml | 26 + .../databases/charts/kafka/templates/svc.yaml | 49 + .../charts/kafka/values-production.yaml | 931 +++++++++++++++ .../helm/databases/charts/kafka/values.yaml | 934 +++++++++++++++ scripts/helm/helm/README.md | 14 + scripts/helm/helm/databases/.helmignore | 23 + scripts/helm/helm/databases/Chart.yaml | 42 + .../databases/charts/clickhouse/.helmignore | 23 + .../databases/charts/clickhouse/Chart.yaml | 23 + .../charts/clickhouse/templates/_helpers.tpl | 62 + .../charts/clickhouse/templates/service.yaml | 19 + .../clickhouse/templates/serviceaccount.yaml | 12 + .../clickhouse/templates/statefulset.yaml | 69 ++ .../databases/charts/clickhouse/values.yaml | 62 + .../helm/databases/charts/kafka/.helmignore | 21 + .../helm/databases/charts/kafka/Chart.yaml | 22 + .../helm/databases/charts/kafka/README.md | 737 ++++++++++++ .../charts/kafka/charts/zookeeper/.helmignore | 21 + .../charts/kafka/charts/zookeeper/Chart.yaml | 20 + .../charts/kafka/charts/zookeeper/README.md | 297 +++++ .../charts/zookeeper/templates/NOTES.txt | 57 + .../charts/zookeeper/templates/_helpers.tpl | 212 ++++ .../charts/zookeeper/templates/configmap.yaml | 17 + .../zookeeper/templates/metrics-svc.yaml | 29 + .../zookeeper/templates/networkpolicy.yaml | 43 + .../templates/poddisruptionbudget.yaml | 21 + .../zookeeper/templates/prometheusrules.yaml | 27 + .../charts/zookeeper/templates/secrets.yaml | 18 + .../zookeeper/templates/serviceaccount.yaml | 15 + .../zookeeper/templates/servicemonitor.yaml | 38 + .../zookeeper/templates/statefulset.yaml | 334 ++++++ .../zookeeper/templates/svc-headless.yaml | 42 + .../kafka/charts/zookeeper/templates/svc.yaml | 40 + .../charts/zookeeper/values-production.yaml | 430 +++++++ .../charts/kafka/charts/zookeeper/values.yaml | 430 +++++++ .../charts/kafka/files/jks/README.md | 10 + .../helm/databases/charts/kafka/kafka.yaml | 521 +++++++++ .../databases/charts/kafka/requirements.lock | 6 + .../databases/charts/kafka/requirements.yaml | 5 + .../charts/kafka/templates/NOTES.txt | 181 +++ .../charts/kafka/templates/_helpers.tpl | 556 +++++++++ .../charts/kafka/templates/configmap.yaml | 16 + .../charts/kafka/templates/extra-list.yaml | 5 + .../charts/kafka/templates/jaas-secret.yaml | 32 + .../charts/kafka/templates/jks-secret.yaml | 19 + .../charts/kafka/templates/jmx-configmap.yaml | 59 + .../kafka/templates/jmx-metrics-svc.yaml | 45 + .../templates/kafka-metrics-deployment.yaml | 87 ++ .../kafka/templates/kafka-metrics-svc.yaml | 45 + .../kafka/templates/log4j-configmap.yaml | 16 + .../kafka/templates/poddisruptionbudget.yaml | 25 + .../charts/kafka/templates/role.yaml | 23 + .../charts/kafka/templates/rolebinding.yaml | 22 + .../kafka/templates/scripts-configmap.yaml | 118 ++ .../kafka/templates/serviceaccount.yaml | 14 + .../templates/servicemonitor-jmx-metrics.yaml | 36 + .../templates/servicemonitor-metrics.yaml | 36 + .../charts/kafka/templates/statefulset.yaml | 435 +++++++ .../kafka/templates/svc-external-access.yaml | 52 + .../charts/kafka/templates/svc-headless.yaml | 26 + .../databases/charts/kafka/templates/svc.yaml | 49 + .../charts/kafka/values-production.yaml | 931 +++++++++++++++ .../helm/databases/charts/kafka/values.yaml | 934 +++++++++++++++ .../helm/databases/charts/minio/.helmignore | 21 + .../helm/databases/charts/minio/Chart.yaml | 24 + .../helm/databases/charts/minio/README.md | 259 +++++ .../charts/minio/ci/values-production.yaml | 27 + .../charts/minio/templates/NOTES.txt | 71 ++ .../charts/minio/templates/_helpers.tpl | 265 +++++ .../templates/deployment-standalone.yaml | 160 +++ .../charts/minio/templates/ingress.yaml | 43 + .../charts/minio/templates/networkpolicy.yaml | 23 + .../minio/templates/pvc-standalone.yaml | 17 + .../charts/minio/templates/secrets.yaml | 12 + .../charts/minio/templates/service.yaml | 24 + .../minio/templates/serviceaccount.yaml | 10 + .../charts/minio/templates/statefulset.yaml | 181 +++ .../charts/minio/templates/svc-headless.yaml | 16 + .../charts/minio/templates/tls-secrets.yaml | 14 + .../charts/minio/values-production.yaml | 389 +++++++ .../helm/databases/charts/minio/values.yaml | 391 +++++++ .../databases/charts/postgresql/.helmignore | 21 + .../databases/charts/postgresql/Chart.yaml | 26 + .../databases/charts/postgresql/README.md | 707 ++++++++++++ .../postgresql/charts/common/.helmignore | 22 + .../postgresql/charts/common/Chart.yaml | 22 + .../charts/postgresql/charts/common/README.md | 286 +++++ .../charts/common/templates/_affinities.tpl | 94 ++ .../charts/common/templates/_capabilities.tpl | 33 + .../charts/common/templates/_errors.tpl | 20 + .../charts/common/templates/_images.tpl | 43 + .../charts/common/templates/_labels.tpl | 18 + .../charts/common/templates/_names.tpl | 32 + .../charts/common/templates/_secrets.tpl | 49 + .../charts/common/templates/_storage.tpl | 23 + .../charts/common/templates/_tplvalues.tpl | 13 + .../charts/common/templates/_utils.tpl | 45 + .../charts/common/templates/_validations.tpl | 278 +++++ .../charts/common/templates/_warnings.tpl | 14 + .../postgresql/charts/common/values.yaml | 3 + .../postgresql/ci/commonAnnotations.yaml | 3 + .../charts/postgresql/ci/default-values.yaml | 1 + .../ci/shmvolume-disabled-values.yaml | 2 + .../charts/postgresql/files/README.md | 1 + .../charts/postgresql/files/conf.d/README.md | 4 + .../docker-entrypoint-initdb.d/README.md | 3 + .../charts/postgresql/postgresql.yaml | 185 +++ .../charts/postgresql/requirements.lock | 6 + .../charts/postgresql/requirements.yaml | 4 + .../charts/postgresql/templates/NOTES.txt | 59 + .../charts/postgresql/templates/_helpers.tpl | 488 ++++++++ .../postgresql/templates/configmap.yaml | 26 + .../templates/extended-config-configmap.yaml | 21 + .../postgresql/templates/extra-list.yaml | 5 + .../templates/initialization-configmap.yaml | 24 + .../templates/metrics-configmap.yaml | 13 + .../postgresql/templates/metrics-svc.yaml | 25 + .../postgresql/templates/networkpolicy.yaml | 36 + .../templates/podsecuritypolicy.yaml | 37 + .../postgresql/templates/prometheusrule.yaml | 23 + .../charts/postgresql/templates/role.yaml | 19 + .../postgresql/templates/rolebinding.yaml | 19 + .../charts/postgresql/templates/secrets.yaml | 23 + .../postgresql/templates/serviceaccount.yaml | 11 + .../postgresql/templates/servicemonitor.yaml | 33 + .../templates/statefulset-slaves.yaml | 403 +++++++ .../postgresql/templates/statefulset.yaml | 580 ++++++++++ .../postgresql/templates/svc-headless.yaml | 18 + .../charts/postgresql/templates/svc-read.yaml | 42 + .../charts/postgresql/templates/svc.yaml | 40 + .../charts/postgresql/values-production.yaml | 711 ++++++++++++ .../charts/postgresql/values.schema.json | 103 ++ .../databases/charts/postgresql/values.yaml | 722 ++++++++++++ .../helm/databases/charts/redis/.helmignore | 21 + .../helm/databases/charts/redis/Chart.lock | 6 + .../helm/databases/charts/redis/Chart.yaml | 29 + .../helm/databases/charts/redis/README.md | 738 ++++++++++++ .../charts/redis/charts/common/.helmignore | 22 + .../charts/redis/charts/common/Chart.yaml | 23 + .../charts/redis/charts/common/README.md | 322 ++++++ .../charts/common/templates/_affinities.tpl | 94 ++ .../charts/common/templates/_capabilities.tpl | 95 ++ .../redis/charts/common/templates/_errors.tpl | 23 + .../redis/charts/common/templates/_images.tpl | 47 + .../charts/common/templates/_ingress.tpl | 42 + .../redis/charts/common/templates/_labels.tpl | 18 + .../redis/charts/common/templates/_names.tpl | 32 + .../charts/common/templates/_secrets.tpl | 129 +++ .../charts/common/templates/_storage.tpl | 23 + .../charts/common/templates/_tplvalues.tpl | 13 + .../redis/charts/common/templates/_utils.tpl | 62 + .../charts/common/templates/_warnings.tpl | 14 + .../templates/validations/_cassandra.tpl | 72 ++ .../common/templates/validations/_mariadb.tpl | 103 ++ .../common/templates/validations/_mongodb.tpl | 108 ++ .../templates/validations/_postgresql.tpl | 131 +++ .../common/templates/validations/_redis.tpl | 72 ++ .../templates/validations/_validations.tpl | 46 + .../charts/redis/charts/common/values.yaml | 3 + .../charts/redis/ci/default-values.yaml | 1 + .../charts/redis/ci/extra-flags-values.yaml | 11 + .../redis/ci/production-sentinel-values.yaml | 682 +++++++++++ .../redis/img/redis-cluster-topology.png | Bin 0 -> 11448 bytes .../charts/redis/img/redis-topology.png | Bin 0 -> 9709 bytes .../charts/redis/templates/NOTES.txt | 136 +++ .../charts/redis/templates/_helpers.tpl | 445 ++++++++ .../redis/templates/configmap-scripts.yaml | 430 +++++++ .../charts/redis/templates/configmap.yaml | 53 + .../charts/redis/templates/headless-svc.yaml | 28 + .../redis/templates/health-configmap.yaml | 176 +++ .../redis/templates/metrics-prometheus.yaml | 39 + .../metrics-sentinel-prometheus.yaml | 33 + .../redis/templates/metrics-sentinel-svc.yaml | 35 + .../charts/redis/templates/metrics-svc.yaml | 34 + .../charts/redis/templates/networkpolicy.yaml | 74 ++ .../databases/charts/redis/templates/pdb.yaml | 22 + .../redis/templates/prometheusrule.yaml | 25 + .../databases/charts/redis/templates/psp.yaml | 43 + .../templates/redis-master-statefulset.yaml | 394 +++++++ .../redis/templates/redis-master-svc.yaml | 43 + .../templates/redis-node-statefulset.yaml | 556 +++++++++ .../charts/redis/templates/redis-role.yaml | 22 + .../redis/templates/redis-rolebinding.yaml | 19 + .../redis/templates/redis-serviceaccount.yaml | 15 + .../templates/redis-slave-statefulset.yaml | 398 +++++++ .../redis/templates/redis-slave-svc.yaml | 43 + .../templates/redis-with-sentinel-svc.yaml | 43 + .../charts/redis/templates/secret.yaml | 15 + .../databases/charts/redis/values.schema.json | 168 +++ .../helm/databases/charts/redis/values.yaml | 1008 +++++++++++++++++ .../helm/helm/databases/templates/NOTES.txt | 5 + .../helm/databases/templates/_helpers.tpl | 62 + scripts/helm/helm/databases/values.yaml | 140 +++ scripts/helm/helm/init.sh | 104 ++ scripts/helm/helm/openreplay/.helmignore | 23 + scripts/helm/helm/openreplay/Chart.yaml | 25 + .../helm/openreplay/charts/alerts/.helmignore | 23 + .../helm/openreplay/charts/alerts/Chart.yaml | 24 + .../charts/alerts/templates/NOTES.txt | 22 + .../charts/alerts/templates/_helpers.tpl | 62 + .../charts/alerts/templates/deployment.yaml | 64 ++ .../charts/alerts/templates/hpa.yaml | 28 + .../charts/alerts/templates/ingress.yaml | 61 + .../charts/alerts/templates/service.yaml | 15 + .../alerts/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../helm/openreplay/charts/alerts/values.yaml | 87 ++ .../helm/openreplay/charts/assets/.helmignore | 23 + .../helm/openreplay/charts/assets/Chart.yaml | 24 + .../charts/assets/templates/NOTES.txt | 22 + .../charts/assets/templates/_helpers.tpl | 62 + .../charts/assets/templates/deployment.yaml | 84 ++ .../charts/assets/templates/hpa.yaml | 28 + .../charts/assets/templates/ingress.yaml | 61 + .../charts/assets/templates/service.yaml | 15 + .../assets/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../helm/openreplay/charts/assets/values.yaml | 84 ++ .../openreplay/charts/chalice/.helmignore | 23 + .../helm/openreplay/charts/chalice/Chart.yaml | 24 + .../charts/chalice/templates/NOTES.txt | 22 + .../charts/chalice/templates/_helpers.tpl | 62 + .../charts/chalice/templates/deployment.yaml | 92 ++ .../charts/chalice/templates/hpa.yaml | 28 + .../charts/chalice/templates/ingress.yaml | 61 + .../charts/chalice/templates/service.yaml | 15 + .../chalice/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../openreplay/charts/chalice/values.yaml | 116 ++ .../helm/openreplay/charts/db/.helmignore | 23 + .../helm/helm/openreplay/charts/db/Chart.yaml | 24 + .../openreplay/charts/db/templates/NOTES.txt | 22 + .../charts/db/templates/_helpers.tpl | 62 + .../charts/db/templates/deployment.yaml | 72 ++ .../openreplay/charts/db/templates/hpa.yaml | 28 + .../charts/db/templates/ingress.yaml | 61 + .../charts/db/templates/service.yaml | 15 + .../charts/db/templates/serviceaccount.yaml | 12 + .../db/templates/tests/test-connection.yaml | 15 + .../helm/openreplay/charts/db/values.yaml | 85 ++ .../helm/openreplay/charts/ender/.helmignore | 23 + .../helm/openreplay/charts/ender/Chart.yaml | 24 + .../charts/ender/templates/NOTES.txt | 22 + .../charts/ender/templates/_helpers.tpl | 62 + .../charts/ender/templates/deployment.yaml | 70 ++ .../charts/ender/templates/hpa.yaml | 28 + .../charts/ender/templates/ingress.yaml | 61 + .../charts/ender/templates/service.yaml | 15 + .../ender/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../helm/openreplay/charts/ender/values.yaml | 85 ++ .../helm/openreplay/charts/http/.helmignore | 23 + .../helm/openreplay/charts/http/Chart.yaml | 24 + .../charts/http/templates/NOTES.txt | 22 + .../charts/http/templates/_helpers.tpl | 62 + .../charts/http/templates/deployment.yaml | 80 ++ .../openreplay/charts/http/templates/hpa.yaml | 28 + .../charts/http/templates/ingress.yaml | 61 + .../charts/http/templates/service.yaml | 15 + .../charts/http/templates/serviceaccount.yaml | 12 + .../http/templates/tests/test-connection.yaml | 15 + .../helm/openreplay/charts/http/values.yaml | 89 ++ .../charts/integrations/.helmignore | 23 + .../openreplay/charts/integrations/Chart.yaml | 24 + .../charts/integrations/templates/NOTES.txt | 22 + .../integrations/templates/_helpers.tpl | 62 + .../integrations/templates/deployment.yaml | 72 ++ .../charts/integrations/templates/hpa.yaml | 28 + .../integrations/templates/ingress.yaml | 61 + .../integrations/templates/service.yaml | 15 + .../templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../charts/integrations/values.yaml | 86 ++ .../charts/nginx-ingress/.helmignore | 23 + .../charts/nginx-ingress/Chart.yaml | 24 + .../charts/nginx-ingress/templates/NOTES.txt | 22 + .../nginx-ingress/templates/_helpers.tpl | 62 + .../nginx-ingress/templates/configMap.yaml | 154 +++ .../nginx-ingress/templates/deployment.yaml | 77 ++ .../charts/nginx-ingress/templates/hpa.yaml | 28 + .../nginx-ingress/templates/ingress.yaml | 61 + .../nginx-ingress/templates/secrets.yaml | 9 + .../nginx-ingress/templates/service.yaml | 19 + .../templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../charts/nginx-ingress/values.yaml | 173 +++ .../helm/openreplay/charts/sink/.helmignore | 23 + .../helm/openreplay/charts/sink/Chart.yaml | 24 + .../charts/sink/templates/NOTES.txt | 22 + .../charts/sink/templates/_helpers.tpl | 62 + .../charts/sink/templates/deployment.yaml | 86 ++ .../openreplay/charts/sink/templates/hpa.yaml | 28 + .../charts/sink/templates/ingress.yaml | 61 + .../charts/sink/templates/service.yaml | 15 + .../charts/sink/templates/serviceaccount.yaml | 12 + .../sink/templates/tests/test-connection.yaml | 15 + .../helm/openreplay/charts/sink/values.yaml | 92 ++ .../openreplay/charts/storage/.helmignore | 23 + .../helm/openreplay/charts/storage/Chart.yaml | 24 + .../charts/storage/templates/NOTES.txt | 22 + .../charts/storage/templates/_helpers.tpl | 62 + .../charts/storage/templates/deployment.yaml | 102 ++ .../charts/storage/templates/hpa.yaml | 28 + .../charts/storage/templates/ingress.yaml | 61 + .../charts/storage/templates/service.yaml | 15 + .../storage/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../openreplay/charts/storage/values.yaml | 93 ++ .../openreplay/charts/utilities/.helmignore | 23 + .../openreplay/charts/utilities/Chart.yaml | 24 + .../charts/utilities/templates/NOTES.txt | 22 + .../charts/utilities/templates/_helpers.tpl | 62 + .../utilities/templates/deployment.yaml | 74 ++ .../charts/utilities/templates/hpa.yaml | 28 + .../charts/utilities/templates/ingress.yaml | 61 + .../charts/utilities/templates/service.yaml | 15 + .../utilities/templates/serviceaccount.yaml | 12 + .../templates/tests/test-connection.yaml | 15 + .../openreplay/charts/utilities/values.yaml | 85 ++ .../helm/helm/openreplay/files/clickhouse.sh | 38 + scripts/helm/helm/openreplay/files/dbops.sh | 82 ++ scripts/helm/helm/openreplay/files/kafka.sh | 41 + scripts/helm/helm/openreplay/files/minio.sh | 56 + .../helm/helm/openreplay/files/postgresql.sh | 41 + .../helm/helm/openreplay/templates/NOTES.txt | 22 + .../helm/openreplay/templates/_helpers.tpl | 62 + .../helm/helm/openreplay/templates/job.yaml | 146 +++ .../openreplay/templates/serviceaccount.yaml | 12 + scripts/helm/helm/openreplay/values.yaml | 114 ++ scripts/helm/helm/vars.yaml | 68 ++ 382 files changed, 35672 insertions(+) create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/.helmignore create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml create mode 100644 ee/scripts/helm/helm/databases/charts/clickhouse/values.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/.helmignore create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/Chart.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/README.md create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/files/jks/README.md create mode 100644 ee/scripts/helm/helm/databases/charts/kafka/kafka.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/requirements.lock create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/requirements.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/_helpers.tpl create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/configmap.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/extra-list.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/jaas-secret.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/jks-secret.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/jmx-configmap.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/role.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/values-production.yaml create mode 100755 ee/scripts/helm/helm/databases/charts/kafka/values.yaml create mode 100644 scripts/helm/helm/README.md create mode 100644 scripts/helm/helm/databases/.helmignore create mode 100644 scripts/helm/helm/databases/Chart.yaml create mode 100644 scripts/helm/helm/databases/charts/clickhouse/.helmignore create mode 100644 scripts/helm/helm/databases/charts/clickhouse/Chart.yaml create mode 100644 scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl create mode 100644 scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml create mode 100644 scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml create mode 100644 scripts/helm/helm/databases/charts/clickhouse/values.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/.helmignore create mode 100755 scripts/helm/helm/databases/charts/kafka/Chart.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/README.md create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/files/jks/README.md create mode 100644 scripts/helm/helm/databases/charts/kafka/kafka.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/requirements.lock create mode 100755 scripts/helm/helm/databases/charts/kafka/requirements.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/_helpers.tpl create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/extra-list.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/jaas-secret.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/jks-secret.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/jmx-configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/role.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/templates/svc.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/values-production.yaml create mode 100755 scripts/helm/helm/databases/charts/kafka/values.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/.helmignore create mode 100755 scripts/helm/helm/databases/charts/minio/Chart.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/README.md create mode 100755 scripts/helm/helm/databases/charts/minio/ci/values-production.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/NOTES.txt create mode 100755 scripts/helm/helm/databases/charts/minio/templates/_helpers.tpl create mode 100755 scripts/helm/helm/databases/charts/minio/templates/deployment-standalone.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/ingress.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/networkpolicy.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/pvc-standalone.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/secrets.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/service.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/serviceaccount.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/statefulset.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/svc-headless.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/templates/tls-secrets.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/values-production.yaml create mode 100755 scripts/helm/helm/databases/charts/minio/values.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/.helmignore create mode 100755 scripts/helm/helm/databases/charts/postgresql/Chart.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/README.md create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/.helmignore create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/Chart.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/README.md create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_affinities.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_capabilities.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_errors.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_images.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_labels.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_names.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_secrets.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_storage.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_utils.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_validations.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_warnings.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/charts/common/values.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/ci/commonAnnotations.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/ci/default-values.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/files/README.md create mode 100755 scripts/helm/helm/databases/charts/postgresql/files/conf.d/README.md create mode 100755 scripts/helm/helm/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md create mode 100644 scripts/helm/helm/databases/charts/postgresql/postgresql.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/requirements.lock create mode 100755 scripts/helm/helm/databases/charts/postgresql/requirements.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/NOTES.txt create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/_helpers.tpl create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/extended-config-configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/extra-list.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/initialization-configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/metrics-configmap.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/metrics-svc.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/networkpolicy.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/podsecuritypolicy.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/prometheusrule.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/role.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/rolebinding.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/secrets.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/serviceaccount.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/servicemonitor.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/statefulset-slaves.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/statefulset.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/svc-headless.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/svc-read.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/templates/svc.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/values-production.yaml create mode 100755 scripts/helm/helm/databases/charts/postgresql/values.schema.json create mode 100755 scripts/helm/helm/databases/charts/postgresql/values.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/.helmignore create mode 100644 scripts/helm/helm/databases/charts/redis/Chart.lock create mode 100644 scripts/helm/helm/databases/charts/redis/Chart.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/README.md create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/.helmignore create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/Chart.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/README.md create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_affinities.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_capabilities.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_errors.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_images.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_ingress.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_labels.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_names.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_secrets.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_storage.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_tplvalues.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_utils.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/_warnings.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_redis.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_validations.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/charts/common/values.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/ci/default-values.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/ci/extra-flags-values.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/ci/production-sentinel-values.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/img/redis-cluster-topology.png create mode 100644 scripts/helm/helm/databases/charts/redis/img/redis-topology.png create mode 100644 scripts/helm/helm/databases/charts/redis/templates/NOTES.txt create mode 100644 scripts/helm/helm/databases/charts/redis/templates/_helpers.tpl create mode 100644 scripts/helm/helm/databases/charts/redis/templates/configmap-scripts.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/configmap.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/headless-svc.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/health-configmap.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/metrics-prometheus.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-svc.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/metrics-svc.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/networkpolicy.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/pdb.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/prometheusrule.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/psp.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-master-statefulset.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-master-svc.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-node-statefulset.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-role.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-rolebinding.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-serviceaccount.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-slave-statefulset.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-slave-svc.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/redis-with-sentinel-svc.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/templates/secret.yaml create mode 100644 scripts/helm/helm/databases/charts/redis/values.schema.json create mode 100644 scripts/helm/helm/databases/charts/redis/values.yaml create mode 100644 scripts/helm/helm/databases/templates/NOTES.txt create mode 100644 scripts/helm/helm/databases/templates/_helpers.tpl create mode 100644 scripts/helm/helm/databases/values.yaml create mode 100644 scripts/helm/helm/init.sh create mode 100644 scripts/helm/helm/openreplay/.helmignore create mode 100644 scripts/helm/helm/openreplay/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/alerts/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/alerts/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/assets/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/assets/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/chalice/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/chalice/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/db/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/db/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/ender/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/ender/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/http/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/http/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/integrations/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/integrations/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/configMap.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/secrets.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/nginx-ingress/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/sink/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/sink/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/storage/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/storage/values.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/.helmignore create mode 100644 scripts/helm/helm/openreplay/charts/utilities/Chart.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/deployment.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/hpa.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/ingress.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/service.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/templates/tests/test-connection.yaml create mode 100644 scripts/helm/helm/openreplay/charts/utilities/values.yaml create mode 100644 scripts/helm/helm/openreplay/files/clickhouse.sh create mode 100644 scripts/helm/helm/openreplay/files/dbops.sh create mode 100644 scripts/helm/helm/openreplay/files/kafka.sh create mode 100644 scripts/helm/helm/openreplay/files/minio.sh create mode 100644 scripts/helm/helm/openreplay/files/postgresql.sh create mode 100644 scripts/helm/helm/openreplay/templates/NOTES.txt create mode 100644 scripts/helm/helm/openreplay/templates/_helpers.tpl create mode 100644 scripts/helm/helm/openreplay/templates/job.yaml create mode 100644 scripts/helm/helm/openreplay/templates/serviceaccount.yaml create mode 100644 scripts/helm/helm/openreplay/values.yaml create mode 100644 scripts/helm/helm/vars.yaml diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/.helmignore b/ee/scripts/helm/helm/databases/charts/clickhouse/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml b/ee/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml new file mode 100644 index 000000000..c7a0eb3d6 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: clickhouse +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.16.0 diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl new file mode 100644 index 000000000..44cfadff0 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "clickhouse.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "clickhouse.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "clickhouse.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "clickhouse.labels" -}} +helm.sh/chart: {{ include "clickhouse.chart" . }} +{{ include "clickhouse.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "clickhouse.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clickhouse.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "clickhouse.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "clickhouse.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml new file mode 100644 index 000000000..4496f556c --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + labels: + {{- include "clickhouse.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.webPort }} + targetPort: web + protocol: TCP + name: web + - port: {{ .Values.service.dataPort }} + targetPort: data + protocol: TCP + name: data + selector: + {{- include "clickhouse.selectorLabels" . | nindent 4 }} diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml new file mode 100644 index 000000000..1f1183598 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "clickhouse.serviceAccountName" . }} + labels: + {{- include "clickhouse.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml new file mode 100644 index 000000000..392976eec --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "clickhouse.fullname" . }} + labels: + {{- include "clickhouse.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + serviceName: {{ include "clickhouse.fullname" . }} + selector: + matchLabels: + {{- include "clickhouse.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "clickhouse.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "clickhouse.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + env: + {{- range $key, $value := .Values.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 9000 + name: web + - containerPort: 8123 + name: data + volumeMounts: + - name: ch-volume + mountPath: /var/lib/mydata + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: ch-volume + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.storageSize }} diff --git a/ee/scripts/helm/helm/databases/charts/clickhouse/values.yaml b/ee/scripts/helm/helm/databases/charts/clickhouse/values.yaml new file mode 100644 index 000000000..4cba1c1f8 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/clickhouse/values.yaml @@ -0,0 +1,62 @@ +# Default values for clickhouse. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: yandex/clickhouse-server + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "20.9" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +env: {} + +service: + webPort: 9000 + dataPort: 8123 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} +storageSize: 8G diff --git a/ee/scripts/helm/helm/databases/charts/kafka/.helmignore b/ee/scripts/helm/helm/databases/charts/kafka/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/ee/scripts/helm/helm/databases/charts/kafka/Chart.yaml b/ee/scripts/helm/helm/databases/charts/kafka/Chart.yaml new file mode 100755 index 000000000..165e70d55 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + category: Infrastructure +apiVersion: v1 +appVersion: 2.6.0 +description: Apache Kafka is a distributed streaming platform. +engine: gotpl +home: https://github.com/bitnami/charts/tree/master/bitnami/kafka +icon: https://bitnami.com/assets/stacks/kafka/img/kafka-stack-110x117.png +keywords: +- kafka +- zookeeper +- streaming +- producer +- consumer +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: kafka +sources: +- https://github.com/bitnami/bitnami-docker-kafka +- https://kafka.apache.org/ +version: 11.8.6 diff --git a/ee/scripts/helm/helm/databases/charts/kafka/README.md b/ee/scripts/helm/helm/databases/charts/kafka/README.md new file mode 100755 index 000000000..5584bd43d --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/README.md @@ -0,0 +1,737 @@ +# Kafka + +[Kafka](https://www.kafka.org/) is a distributed streaming platform used for building real-time data pipelines and streaming apps. It is horizontally scalable, fault-tolerant, wicked fast, and runs in production in thousands of companies. + +## TL;DR + +```console +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install my-release bitnami/kafka +``` + +## Introduction + +This chart bootstraps a [Kafka](https://github.com/bitnami/bitnami-docker-kafka) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install my-release bitnami/kafka +``` + +These commands deploy Kafka on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +The following tables lists the configurable parameters of the Kafka chart and their default values per section/component: + +### Global parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | + +### Common parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `nameOverride` | String to partially override kafka.fullname | `nil` | +| `fullnameOverride` | String to fully override kafka.fullname | `nil` | +| `clusterDomain` | Default Kubernetes cluster domain | `cluster.local` | +| `commonLabels` | Labels to add to all deployed objects | `{}` | +| `commonAnnotations` | Annotations to add to all deployed objects | `{}` | +| `extraDeploy` | Array of extra objects to deploy with the release | `nil` (evaluated as a template) | + +### Kafka parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `image.registry` | Kafka image registry | `docker.io` | +| `image.repository` | Kafka image name | `bitnami/kafka` | +| `image.tag` | Kafka image tag | `{TAG_NAME}` | +| `image.pullPolicy` | Kafka image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Set to true if you would like to see extra information on logs | `false` | +| `config` | Configuration file for Kafka. Auto-generated based on other parameters when not specified | `nil` | +| `existingConfigmap` | Name of existing ConfigMap with Kafka configuration | `nil` | +| `log4j` | An optional log4j.properties file to overwrite the default of the Kafka brokers. | `nil` | +| `existingLog4jConfigMap` | The name of an existing ConfigMap containing a log4j.properties file. | `nil` | +| `heapOpts` | Kafka's Java Heap size | `-Xmx1024m -Xms1024m` | +| `deleteTopicEnable` | Switch to enable topic deletion or not | `false` | +| `autoCreateTopicsEnable` | Switch to enable auto creation of topics. Enabling auto creation of topics not recommended for production or similar environments | `false` | +| `logFlushIntervalMessages` | The number of messages to accept before forcing a flush of data to disk | `10000` | +| `logFlushIntervalMs` | The maximum amount of time a message can sit in a log before we force a flush | `1000` | +| `logRetentionBytes` | A size-based retention policy for logs | `_1073741824` | +| `logRetentionCheckIntervalMs` | The interval at which log segments are checked to see if they can be deleted | `300000` | +| `logRetentionHours` | The minimum age of a log file to be eligible for deletion due to age | `168` | +| `logSegmentBytes` | The maximum size of a log segment file. When this size is reached a new log segment will be created | `_1073741824` | +| `logsDirs` | A comma separated list of directories under which to store log files | `/bitnami/kafka/data` | +| `maxMessageBytes` | The largest record batch size allowed by Kafka | `1000012` | +| `defaultReplicationFactor` | Default replication factors for automatically created topics | `1` | +| `offsetsTopicReplicationFactor` | The replication factor for the offsets topic | `1` | +| `transactionStateLogReplicationFactor` | The replication factor for the transaction topic | `1` | +| `transactionStateLogMinIsr` | Overridden min.insync.replicas config for the transaction topic | `1` | +| `numIoThreads` | The number of threads doing disk I/O | `8` | +| `numNetworkThreads` | The number of threads handling network requests | `3` | +| `numPartitions` | The default number of log partitions per topic | `1` | +| `numRecoveryThreadsPerDataDir` | The number of threads per data directory to be used for log recovery at startup and flushing at shutdown | `1` | +| `socketReceiveBufferBytes` | The receive buffer (SO_RCVBUF) used by the socket server | `102400` | +| `socketRequestMaxBytes` | The maximum size of a request that the socket server will accept (protection against OOM) | `_104857600` | +| `socketSendBufferBytes` | The send buffer (SO_SNDBUF) used by the socket server | `102400` | +| `zookeeperConnectionTimeoutMs` | Timeout in ms for connecting to Zookeeper | `6000` | +| `extraEnvVars` | Extra environment variables to add to kafka pods | `[]` | +| `extraVolumes` | Extra volume(s) to add to Kafka statefulset | `[]` | +| `extraVolumeMounts` | Extra volumeMount(s) to add to Kafka containers | `[]` | +| `auth.clientProtocol` | Authentication protocol for communications with clients. Allowed protocols: `plaintext`, `tls`, `mtls`, `sasl` and `sasl_tls` | `plaintext` | +| `auth.interBrokerProtocol` | Authentication protocol for inter-broker communications. Allowed protocols: `plaintext`, `tls`, `mtls`, `sasl` and `sasl_tls` | `plaintext` | +| `auth.saslMechanisms` | SASL mechanisms when either `auth.interBrokerProtocol` or `auth.clientProtocol` are `sasl`. Allowed types: `plain`, `scram-sha-256`, `scram-sha-512` | `plain,scram-sha-256,scram-sha-512` | +| `auth.saslInterBrokerMechanism` | SASL mechanism to use as inter broker protocol, it must be included at `auth.saslMechanisms` | `plain` | +| `auth.jksSecret` | Name of the existing secret containing the truststore and one keystore per Kafka broker you have in the cluster | `nil` | +| `auth.jksPassword` | Password to access the JKS files when they are password-protected | `nil` | +| `auth.tlsEndpointIdentificationAlgorithm` | The endpoint identification algorithm to validate server hostname using server certificate | `https` | +| `auth.jaas.interBrokerUser` | Kafka inter broker communication user for SASL authentication | `admin` | +| `auth.jaas.interBrokerPassword` | Kafka inter broker communication password for SASL authentication | `nil` | +| `auth.jaas.zookeeperUser` | Kafka Zookeeper user for SASL authentication | `nil` | +| `auth.jaas.zookeeperPassword` | Kafka Zookeeper password for SASL authentication | `nil` | +| `auth.jaas.existingSecret` | Name of the existing secret containing credentials for brokerUser, interBrokerUser and zookeeperUser | `nil` | +| `auth.jaas.clientUsers` | List of Kafka client users to be created, separated by commas. This values will override `auth.jaas.clientUser` | `[]` | +| `auth.jaas.clientPasswords` | List of passwords for `auth.jaas.clientUsers`. It is mandatory to provide the passwords when using `auth.jaas.clientUsers` | `[]` | +| `listeners` | The address(es) the socket server listens on. Auto-calculated it's set to an empty array | `[]` | +| `advertisedListeners` | The address(es) (hostname:port) the broker will advertise to producers and consumers. Auto-calculated it's set to an empty array | `[]` | +| `listenerSecurityProtocolMap` | The protocol->listener mapping. Auto-calculated it's set to nil | `nil` | +| `allowPlaintextListener` | Allow to use the PLAINTEXT listener | `true` | +| `interBrokerListenerName` | The listener that the brokers should communicate on | `INTERNAL` | + +### Statefulset parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | Number of Kafka nodes | `1` | +| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `podLabels` | Kafka pod labels | `{}` (evaluated as a template) | +| `podAnnotations` | Kafka Pod annotations | `{}` (evaluated as a template) | +| `affinity` | Affinity for pod assignment | `{}` (evaluated as a template) | +| `priorityClassName` | Name of the existing priority class to be used by kafka pods | `""` | +| `nodeSelector` | Node labels for pod assignment | `{}` (evaluated as a template) | +| `tolerations` | Tolerations for pod assignment | `[]` (evaluated as a template) | +| `podSecurityContext` | Kafka pods' Security Context | `{}` | +| `containerSecurityContext` | Kafka containers' Security Context | `{}` | +| `resources.limits` | The resources limits for Kafka containers | `{}` | +| `resources.requests` | The requested resources for Kafka containers | `{}` | +| `livenessProbe` | Liveness probe configuration for Kafka | `Check values.yaml file` | +| `readinessProbe` | Readiness probe configuration for Kafka | `Check values.yaml file` | +| `customLivenessProbe` | Custom Liveness probe configuration for Kafka | `{}` | +| `customReadinessProbe` | Custom Readiness probe configuration for Kafka | `{}` | +| `pdb.create` | Enable/disable a Pod Disruption Budget creation | `false` | +| `pdb.minAvailable` | Minimum number/percentage of pods that should remain scheduled | `nil` | +| `pdb.maxUnavailable` | Maximum number/percentage of pods that may be made unavailable | `1` | +| `command` | Override kafka container command | `['/scripts/setup.sh']` (evaluated as a template) | +| `args` | Override kafka container arguments | `[]` (evaluated as a template) | +| `sidecars` | Attach additional sidecar containers to the Kafka pod | `{}` | + +### Exposure parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | Kafka port for client connections | `9092` | +| `service.internalPort` | Kafka port for inter-broker connections | `9093` | +| `service.externalPort` | Kafka port for external connections | `9094` | +| `service.nodePorts.client` | Nodeport for client connections | `""` | +| `service.nodePorts.external` | Nodeport for external connections | `""` | +| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | +| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]` | +| `service.annotations` | Service annotations | `{}`(evaluated as a template) | +| `externalAccess.enabled` | Enable Kubernetes external cluster access to Kafka brokers | `false` | +| `externalAccess.autoDiscovery.enabled` | Enable using an init container to auto-detect external IPs/ports by querying the K8s API | `false` | +| `externalAccess.autoDiscovery.image.registry` | Init container auto-discovery image registry (kubectl) | `docker.io` | +| `externalAccess.autoDiscovery.image.repository` | Init container auto-discovery image name (kubectl) | `bitnami/kubectl` | +| `externalAccess.autoDiscovery.image.tag` | Init container auto-discovery image tag (kubectl) | `{TAG_NAME}` | +| `externalAccess.autoDiscovery.image.pullPolicy` | Init container auto-discovery image pull policy (kubectl) | `Always` | +| `externalAccess.autoDiscovery.resources.limits` | Init container auto-discovery resource limits | `{}` | +| `externalAccess.autoDiscovery.resources.requests` | Init container auto-discovery resource requests | `{}` | +| `externalAccess.service.type` | Kubernetes Service type for external access. It can be NodePort or LoadBalancer | `LoadBalancer` | +| `externalAccess.service.port` | Kafka port used for external access when service type is LoadBalancer | `9094` | +| `externalAccess.service.loadBalancerIPs` | Array of load balancer IPs for Kafka brokers | `[]` | +| `externalAccess.service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]` | +| `externalAccess.service.domain` | Domain or external ip used to configure Kafka external listener when service type is NodePort | `nil` | +| `externalAccess.service.nodePorts` | Array of node ports used to configure Kafka external listener when service type is NodePort | `[]` | +| `externalAccess.service.annotations` | Service annotations for external access | `{}`(evaluated as a template) | + +### Persistence parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `persistence.enabled` | Enable Kafka data persistence using PVC, note that Zookeeper persistence is unaffected | `true` | +| `persistence.existingClaim` | Provide an existing `PersistentVolumeClaim`, the value is evaluated as a template | `nil` | +| `persistence.storageClass` | PVC Storage Class for Kafka data volume | `nil` | +| `persistence.accessMode` | PVC Access Mode for Kafka data volume | `ReadWriteOnce` | +| `persistence.size` | PVC Storage Request for Kafka data volume | `8Gi` | +| `persistence.annotations` | Annotations for the PVC | `{}`(evaluated as a template) | + +### RBAC parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `serviceAccount.create` | Enable creation of ServiceAccount for Kafka pods | `true` | +| `serviceAccount.name` | Name of the created serviceAccount | Generated using the `kafka.fullname` template | +| `rbac.create` | Weather to create & use RBAC resources or not | `false` | + +### Volume Permissions parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `volumePermissions.enabled` | Enable init container that changes the owner and group of the persistent volume(s) mountpoint to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/minideb` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `buster` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `volumePermissions.resources.limits` | Init container volume-permissions resource limits | `{}` | +| `volumePermissions.resources.requests` | Init container volume-permissions resource requests | `{}` | + +### Metrics parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `metrics.kafka.enabled` | Whether or not to create a standalone Kafka exporter to expose Kafka metrics | `false` | +| `metrics.kafka.image.registry` | Kafka exporter image registry | `docker.io` | +| `metrics.kafka.image.repository` | Kafka exporter image name | `bitnami/kafka-exporter` | +| `metrics.kafka.image.tag` | Kafka exporter image tag | `{TAG_NAME}` | +| `metrics.kafka.image.pullPolicy` | Kafka exporter image pull policy | `IfNotPresent` | +| `metrics.kafka.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.kafka.extraFlags` | Extra flags to be passed to Kafka exporter | `{}` | +| `metrics.kafka.certificatesSecret` | Name of the existing secret containing the optional certificate and key files | `nil` | +| `metrics.kafka.resources.limits` | Kafka Exporter container resource limits | `{}` | +| `metrics.kafka.resources.requests` | Kafka Exporter container resource requests | `{}` | +| `metrics.kafka.service.type` | Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for Kafka Exporter | `ClusterIP` | +| `metrics.kafka.service.port` | Kafka Exporter Prometheus port | `9308` | +| `metrics.kafka.service.nodePort` | Kubernetes HTTP node port | `""` | +| `metrics.kafka.service.annotations` | Annotations for Prometheus metrics service | `Check values.yaml file` | +| `metrics.kafka.service.loadBalancerIP` | loadBalancerIP if service type is `LoadBalancer` | `nil` | +| `metrics.kafka.service.clusterIP` | Static clusterIP or None for headless services | `nil` | +| `metrics.jmx.enabled` | Whether or not to expose JMX metrics to Prometheus | `false` | +| `metrics.jmx.image.registry` | JMX exporter image registry | `docker.io` | +| `metrics.jmx.image.repository` | JMX exporter image name | `bitnami/jmx-exporter` | +| `metrics.jmx.image.tag` | JMX exporter image tag | `{TAG_NAME}` | +| `metrics.jmx.image.pullPolicy` | JMX exporter image pull policy | `IfNotPresent` | +| `metrics.jmx.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.jmx.resources.limits` | JMX Exporter container resource limits | `{}` | +| `metrics.jmx.resources.requests` | JMX Exporter container resource requests | `{}` | +| `metrics.jmx.service.type` | Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for JMX Exporter | `ClusterIP` | +| `metrics.jmx.service.port` | JMX Exporter Prometheus port | `5556` | +| `metrics.jmx.service.nodePort` | Kubernetes HTTP node port | `""` | +| `metrics.jmx.service.annotations` | Annotations for Prometheus metrics service | `Check values.yaml file` | +| `metrics.jmx.service.loadBalancerIP` | loadBalancerIP if service type is `LoadBalancer` | `nil` | +| `metrics.jmx.service.clusterIP` | Static clusterIP or None for headless services | `nil` | +| `metrics.jmx.whitelistObjectNames` | Allows setting which JMX objects you want to expose to via JMX stats to JMX Exporter | (see `values.yaml`) | +| `metrics.jmx.config` | Configuration file for JMX exporter | (see `values.yaml`) | +| `metrics.jmx.existingConfigmap` | Name of existing ConfigMap with JMX exporter configuration | `nil` | +| `metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.kafka.enabled` or `metrics.jmx.enabled` to be `true`) | `false` | +| `metrics.serviceMonitor.namespace` | Namespace which Prometheus is running in | `monitoring` | +| `metrics.serviceMonitor.interval` | Interval at which metrics should be scraped | `nil` | +| `metrics.serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `nil` (Prometheus Operator default value) | +| `metrics.serviceMonitor.selector` | ServiceMonitor selector labels | `nil` (Prometheus Operator default value) | + +### Zookeeper chart parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `zookeeper.enabled` | Switch to enable or disable the Zookeeper helm chart | `true` | +| `zookeeper.persistence.enabled` | Enable Zookeeper persistence using PVC | `true` | +| `externalZookeeper.servers` | Server or list of external Zookeeper servers to use | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +helm install my-release \ + --set replicaCount=3 \ + bitnami/kafka +``` + +The above command deploys Kafka with 3 brokers (replicas). + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +helm install my-release -f values.yaml bitnami/kafka +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Production configuration and horizontal scaling + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one. + +- Number of Kafka nodes: + +```diff +- replicaCount: 1 ++ replicaCount: 3 +``` + +- Allow to use the PLAINTEXT listener: + +```diff +- allowPlaintextListener: true ++ allowPlaintextListener: false +``` + +- Default replication factors for automatically created topics: + +```diff +- defaultReplicationFactor: 1 ++ defaultReplicationFactor: 3 +``` + +- Allow auto creation of topics. + +```diff +- autoCreateTopicsEnable: true ++ autoCreateTopicsEnable: false +``` + +- The replication factor for the offsets topic: + +```diff +- offsetsTopicReplicationFactor: 1 ++ offsetsTopicReplicationFactor: 3 +``` + +- The replication factor for the transaction topic: + +```diff +- transactionStateLogReplicationFactor: 1 ++ transactionStateLogReplicationFactor: 3 +``` + +- Overridden min.insync.replicas config for the transaction topic: + +```diff +- transactionStateLogMinIsr: 1 ++ transactionStateLogMinIsr: 3 +``` + +- Switch to enable the Kafka SASAL authentication on client and inter-broker communications: + +```diff +- auth.clientProtocol: plaintext ++ auth.clientProtocol: sasl +- auth.interBrokerProtocol: plaintext ++ auth.interBrokerProtocol: sasl +``` + +- Enable Zookeeper authentication: + +```diff ++ auth.jaas.zookeeperUser: zookeeperUser ++ auth.jaas.zookeeperPassword: zookeeperPassword +- zookeeper.auth.enabled: false ++ zookeeper.auth.enabled: true ++ zookeeper.auth.clientUser: zookeeperUser ++ zookeeper.auth.clientPassword: zookeeperPassword ++ zookeeper.auth.serverUsers: zookeeperUser ++ zookeeper.auth.serverPasswords: zookeeperPassword +``` + +- Enable Pod Disruption Budget: + +```diff +- pdb.create: false ++ pdb.create: true +``` + +- Create a separate Kafka metrics exporter: + +```diff +- metrics.kafka.enabled: false ++ metrics.kafka.enabled: true +``` + +- Expose JMX metrics to Prometheus: + +```diff +- metrics.jmx.enabled: false ++ metrics.jmx.enabled: true +``` + +- Enable Zookeeper metrics: + +```diff ++ zookeeper.metrics.enabled: true +``` + +To horizontally scale this chart once it has been deployed, you can upgrade the statefulset using a new value for the `replicaCount` parameter. Please note that, when enabling TLS encryption, you must update your JKS secret including the keystore for the new replicas. + +### Setting custom parameters + +Any environment variable beginning with `KAFKA_CFG_` will be mapped to its corresponding Kafka key. For example, use `KAFKA_CFG_BACKGROUND_THREADS` in order to set `background.threads`. In order to pass custom environment variables use the `extraEnvVars` property. + +### Listeners configuration + +This chart allows you to automatically configure Kafka with 3 listeners: + +- One for inter-broker communications. +- A second one for communications with clients within the K8s cluster. +- (optional) a third listener for communications with clients outside the K8s cluster. Check [this section](#accessing-kafka-brokers-from-outside-the-clusters) for more information. + +For more complex configurations, set the `listeners`, `advertisedListeners` and `listenerSecurityProtocolMap` parameters as needed. + +### Enable security for Kafka and Zookeeper + +You can configure different authentication protocols for each listener you configure in Kafka. For instance, you can use `sasl_tls` authentication for client communications, while using `tls` for inter-broker communications. This table shows the available protocols and the security they provide: + +| Method | Authentication | Encryption via TLS | +|-----------|-------------------------------|--------------------| +| plaintext | None | No | +| tls | None | Yes | +| mtls | Yes (two-way authentication) | Yes | +| sasl | Yes (via SASL) | No | +| sasl_tls | Yes (via SASL) | Yes | + +If you enabled SASL authentication on any listener, you can set the SASL credentials using the parameters below: + +- `auth.jaas.clientUsers`/`auth.jaas.clientPasswords`: when enabling SASL authentication for communications with clients. +- `auth.jaas.interBrokerUser`/`auth.jaas.interBrokerPassword`: when enabling SASL authentication for inter-broker communications. +- `auth.jaas.zookeeperUser`/`auth.jaas.zookeeperPassword`: In the case that the Zookeeper chart is deployed with SASL authentication enabled. + +In order to configure TLS authentication/encryption, you **must** create a secret containing the Java Key Stores (JKS) files: the truststore (`kafka.truststore.jks`) and one keystore (`kafka.keystore.jks`) per Kafka broker you have in the cluster. Then, you need pass the secret name with the `--auth.jksSecret` parameter when deploying the chart. + +> **Note**: If the JKS files are password protected (recommended), you will need to provide the password to get access to the keystores. To do so, use the `auth.jksPassword` parameter to provide your password. + +For instance, to configure TLS authentication on a Kafka cluster with 2 Kafka brokers use the command below to create the secret: + +```console +kubectl create secret generic kafka-jks --from-file=./kafka.truststore.jks --from-file=./kafka-0.keystore.jks --from-file=./kafka-1.keystore.jks +``` + +> **Note**: the command above assumes you already created the trustore and keystores files. This [script](https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh) can help you with the JKS files generation. + +As an alternative to manually create the secret before installing the chart, you can put your JKS files inside the chart folder `files/jks`, an a secret including them will be generated. Please note this alternative requires to have the chart downloaded locally, so you will have to clone this repository or fetch the chart before installing it. + +You can deploy the chart with authentication using the following parameters: + +```console +replicaCount=2 +auth.clientProtocol=sasl +auth.interBrokerProtocol=tls +auth.certificatesSecret=kafka-jks +auth.certificatesPassword=jksPassword +auth.jaas.clientUsers[0]=brokerUser +auth.jaas.clientPassword[0]=brokerPassword +auth.jaas.zookeeperUser=zookeeperUser +auth.jaas.zookeeperPassword=zookeeperPassword +zookeeper.auth.enabled=true +zookeeper.auth.serverUsers=zookeeperUser +zookeeper.auth.serverPasswords=zookeeperPassword +zookeeper.auth.clientUser=zookeeperUser +zookeeper.auth.clientPassword=zookeeperPassword +``` + +If you also enable exposing metrics using the Kafka expoter, and you are using `sasl_tls`, `tls`, or `mtls` authentication protocols, you need to mount the CA certificated used to sign the brokers certificates in the exporter so it can validate the Kafka brokers. To do so, create a secret containing the CA, and set the `metrics.certificatesSecret` parameter. As an alternative, you can skip TLS validation using extra flags: + +```console +metrics.kafka.extraFlags={tls.insecure-skip-tls-verify: ""} +``` + +### Accessing Kafka brokers from outside the cluster + +In order to access Kafka Brokers from outside the cluster, an additional listener and advertised listener must be configured. Additionally, a specific service per kafka pod will be created. + +There are two ways of configuring external access. Using LoadBalancer services or using NodePort services. + +#### Using LoadBalancer services + +You have two alternatives to use LoadBalancer services: + +- Option A) Use random load balancer IPs using an **initContainer** that waits for the IPs to be ready and discover them automatically. + +```console +externalAccess.enabled=true +externalAccess.service.type=LoadBalancer +externalAccess.service.port=9094 +externalAccess.autoDiscovery.enabled=true +serviceAccount.create=true +rbac.create=true +``` + +Note: This option requires creating RBAC rules on clusters where RBAC policies are enabled. + +- Option B) Manually specify the load balancer IPs: + +```console +externalAccess.enabled=true +externalAccess.service.type=LoadBalancer +externalAccess.service.port=9094 +externalAccess.service.loadBalancerIPs[0]='external-ip-1' +externalAccess.service.loadBalancerIPs[1]='external-ip-2'} +``` + +Note: You need to know in advance the load balancer IPs so each Kafka broker advertised listener is configured with it. + +#### Using NodePort services + +You have two alternatives to use NodePort services: + +- Option A) Use random node ports using an **initContainer** that discover them automatically. + +```console +externalAccess.enabled=true +externalAccess.service.type=NodePort +externalAccess.autoDiscovery.enabled=true +serviceAccount.create=true +rbac.create=true +``` + +Note: This option requires creating RBAC rules on clusters where RBAC policies are enabled. + +- Option B) Manually specify the node ports: + +```console +externalAccess.enabled=true +externalAccess.service.type=NodePort +externalAccess.serivce.nodePorts[0]='node-port-1' +externalAccess.serivce.nodePorts[1]='node-port-2' +``` + +Note: You need to know in advance the node ports that will be exposed so each Kafka broker advertised listener is configured with it. + +The pod will try to get the external ip of the node using `curl -s https://ipinfo.io/ip` unless `externalAccess.service.domain` is provided. + +Following the aforementioned steps will also allow to connect the brokers from the outside using the cluster's default service (when `service.type` is `LoadBalancer` or `NodePort`). Use the property `service.externalPort` to specify the port used for external connections. + +### Sidecars + +If you have a need for additional containers to run within the same pod as Kafka (e.g. an additional metrics or logging exporter), you can do so via the `sidecars` config parameter. Simply define your container according to the Kubernetes container spec. + +```yaml +sidecars: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +``` + +### Deploying extra resources + +There are cases where you may want to deploy extra objects, such as Kafka Connect. For covering this case, the chart allows adding the full specification of other objects using the `extraDeploy` parameter. The following example would create a deployment including a Kafka Connect deployment so you can connect Kafka with MongoDB: + +```yaml +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: |- + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "kafka.fullname" . }}-connect + labels: {{- include "kafka.labels" . | nindent 6 }} + app.kubernetes.io/component: connector + spec: + replicas: 1 + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 8 }} + app.kubernetes.io/component: connector + template: + metadata: + labels: {{- include "kafka.labels" . | nindent 10 }} + app.kubernetes.io/component: connector + spec: + containers: + - name: connect + image: KAFKA-CONNECT-IMAGE + imagePullPolicy: IfNotPresent + ports: + - name: connector + containerPort: 8083 + volumeMounts: + - name: configuration + mountPath: /opt/bitnami/kafka/config + volumes: + - name: configuration + configMap: + name: {{ include "kafka.fullname" . }}-connect + - apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ include "kafka.fullname" . }}-connect + labels: {{- include "kafka.labels" . | nindent 6 }} + app.kubernetes.io/component: connector + data: + connect-standalone.properties: |- + bootstrap.servers = {{ include "kafka.fullname" . }}-0.{{ include "kafka.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.service.port }} + ... + mongodb.properties: |- + connection.uri=mongodb://root:password@mongodb-hostname:27017 + ... + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kafka.fullname" . }}-connect + labels: {{- include "kafka.labels" . | nindent 6 }} + app.kubernetes.io/component: connector + spec: + ports: + - protocol: TCP + port: 8083 + targetPort: connector + selector: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: connector +``` + +You can create the Kafka Connect image using the Dockerfile below: + +```Dockerfile +FROM bitnami/kafka:latest +# Download MongoDB Connector for Apache Kafka https://www.confluent.io/hub/mongodb/kafka-connect-mongodb +RUN mkdir -p /opt/bitnami/kafka/plugins && \ + cd /opt/bitnami/kafka/plugins && \ + curl --remote-name --location --silent https://search.maven.org/remotecontent?filepath=org/mongodb/kafka/mongo-kafka-connect/1.2.0/mongo-kafka-connect-1.2.0-all.jar +CMD /opt/bitnami/kafka/bin/connect-standalone.sh /opt/bitnami/kafka/config/connect-standalone.properties /opt/bitnami/kafka/config/mongo.properties +``` + +## Persistence + +The [Bitnami Kafka](https://github.com/bitnami/bitnami-docker-kafka) image stores the Kafka data at the `/bitnami/kafka` path of the container. + +Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. See the [Parameters](#persistence-parameters) section to configure the PVC or to disable persistence. + +### Adjust permissions of persistent volume mountpoint + +As the image run as non-root by default, it is necessary to adjust the ownership of the persistent volume so that the container can write data into it. + +By default, the chart is configured to use Kubernetes Security Context to automatically change the ownership of the volume. However, this feature does not work in all Kubernetes distributions. +As an alternative, this chart supports using an initContainer to change the ownership of the volume before mounting it in the final destination. + +You can enable this initContainer by setting `volumePermissions.enabled` to `true`. + +## Upgrading + +### To 11.8.0 + +External access to brokers can now be archived through the cluster's Kafka service. + +- `service.nodePort` -> deprecated in favor of `service.nodePorts.client` and `service.nodePorts.external` + +### To 11.7.0 + +The way to configure the users and passwords changed. Now it is allowed to create multiple users during the installation by providing the list of users and passwords. + +- `auth.jaas.clientUser` (string) -> deprecated in favor of `auth.jaas.clientUsers` (array). +- `auth.jaas.clientPassword` (string) -> deprecated in favor of `auth.jaas.clientPasswords` (array). + +### To 11.0.0 + +The way to configure listeners and authentication on Kafka is totally refactored allowing users to configure different authentication protocols on different listeners. Please check the sections [Listeners Configuration](listeners-configuration) and [Listeners Configuration](enable-kafka-for-kafka-and-zookeeper) for more information. + +Backwards compatibility is not guaranteed you adapt your values.yaml to the new format. Here you can find some parameters that were renamed or disappeared in favor of new ones on this major version: + +- `auth.enabled` -> deprecated in favor of `auth.clientProtocol` and `auth.interBrokerProtocol` parameters. +- `auth.ssl` -> deprecated in favor of `auth.clientProtocol` and `auth.interBrokerProtocol` parameters. +- `auth.certificatesSecret` -> renamed to `auth.jksSecret`. +- `auth.certificatesPassword` -> renamed to `auth.jksPassword`. +- `sslEndpointIdentificationAlgorithm` -> renamedo to `auth.tlsEndpointIdentificationAlgorithm`. +- `auth.interBrokerUser` -> renamed to `auth.jaas.interBrokerUser` +- `auth.interBrokerPassword` -> renamed to `auth.jaas.interBrokerPassword` +- `auth.zookeeperUser` -> renamed to `auth.jaas.zookeeperUser` +- `auth.zookeeperPassword` -> renamed to `auth.jaas.zookeeperPassword` +- `auth.existingSecret` -> renamed to `auth.jaas.existingSecret` +- `service.sslPort` -> deprecated in favor of `service.internalPort` +- `service.nodePorts.kafka` and `service.nodePorts.ssl` -> deprecated in favor of `service.nodePort` +- `metrics.kafka.extraFlag` -> new parameter +- `metrics.kafka.certificatesSecret` -> new parameter + +### To 10.0.0 + +If you are setting the `config` or `log4j` parameter, backwards compatibility is not guaranteed, because the `KAFKA_MOUNTED_CONFDIR` has moved from `/opt/bitnami/kafka/conf` to `/bitnami/kafka/config`. In order to continue using these parameters, you must also upgrade your image to `docker.io/bitnami/kafka:2.4.1-debian-10-r38` or later. + +### To 9.0.0 + +Backwards compatibility is not guaranteed you adapt your values.yaml to the new format. Here you can find some parameters that were renamed on this major version: + +```diff +- securityContext.enabled +- securityContext.fsGroup +- securityContext.fsGroup ++ podSecurityContext +- externalAccess.service.loadBalancerIP ++ externalAccess.service.loadBalancerIPs +- externalAccess.service.nodePort ++ externalAccess.service.nodePorts +- metrics.jmx.configMap.enabled +- metrics.jmx.configMap.overrideConfig ++ metrics.jmx.config +- metrics.jmx.configMap.overrideName ++ metrics.jmx.existingConfigmap +``` + +Ports names were prefixed with the protocol to comply with Istio (see https://istio.io/docs/ops/deployment/requirements/). + +### To 8.0.0 + +There is not backwards compatibility since the brokerID changes to the POD_NAME. For more information see [this PR](https://github.com/bitnami/charts/pull/2028). + +### To 7.0.0 + +Backwards compatibility is not guaranteed when Kafka metrics are enabled, unless you modify the labels used on the exporter deployments. +Use the workaround below to upgrade from versions previous to 7.0.0. The following example assumes that the release name is kafka: + +```console +helm upgrade kafka bitnami/kafka --version 6.1.8 --set metrics.kafka.enabled=false +helm upgrade kafka bitnami/kafka --version 7.0.0 --set metrics.kafka.enabled=true +``` + +### To 2.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's deployments. +Use the workaround below to upgrade from versions previous to 2.0.0. The following example assumes that the release name is kafka: + +```console +kubectl delete statefulset kafka-kafka --cascade=false +kubectl delete statefulset kafka-zookeeper --cascade=false +``` + +### To 1.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's deployments. +Use the workaround below to upgrade from versions previous to 1.0.0. The following example assumes that the release name is kafka: + +```console +kubectl delete statefulset kafka-kafka --cascade=false +kubectl delete statefulset kafka-zookeeper --cascade=false +``` diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml new file mode 100755 index 000000000..c3b15dc5c --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml @@ -0,0 +1,20 @@ +annotations: + category: Infrastructure +apiVersion: v1 +appVersion: 3.6.2 +description: A centralized service for maintaining configuration information, naming, + providing distributed synchronization, and providing group services for distributed + applications. +engine: gotpl +home: https://github.com/bitnami/charts/tree/master/bitnami/zookeeper +icon: https://bitnami.com/assets/stacks/zookeeper/img/zookeeper-stack-110x117.png +keywords: +- zookeeper +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: zookeeper +sources: +- https://github.com/bitnami/bitnami-docker-zookeeper +- https://zookeeper.apache.org/ +version: 5.21.9 diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md new file mode 100755 index 000000000..0291875ed --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md @@ -0,0 +1,297 @@ +# ZooKeeper + +[ZooKeeper](https://zookeeper.apache.org/) is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. All of these kinds of services are used in some form or other by distributed applications. + +## TL;DR + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/zookeeper +``` + +## Introduction + +This chart bootstraps a [ZooKeeper](https://github.com/bitnami/bitnami-docker-zookeeper) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/zookeeper +``` + +These commands deploy ZooKeeper on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +The following tables lists the configurable parameters of the ZooKeeper chart and their default values per section/component: + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | + +### Common parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `nameOverride` | String to partially override zookeeper.fullname | `nil` | +| `fullnameOverride` | String to fully override zookeeper.fullname | `nil` | +| `clusterDomain` | Default Kubernetes cluster domain | `cluster.local` | +| `commonLabels` | Labels to add to all deployed objects | `{}` | +| `commonAnnotations` | Annotations to add to all deployed objects | `{}` | +| `schedulerName` | Kubernetes pod scheduler registry | `nil` (use the default-scheduler) | + +### Zookeeper chart parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `image.registry` | ZooKeeper image registry | `docker.io` | +| `image.repository` | ZooKeeper Image name | `bitnami/zookeeper` | +| `image.tag` | ZooKeeper Image tag | `{TAG_NAME}` | +| `image.pullPolicy` | ZooKeeper image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug values should be set | `false` | +| `tickTime` | Basic time unit in milliseconds used by ZooKeeper for heartbeats | `2000` | +| `initLimit` | Time the ZooKeeper servers in quorum have to connect to a leader | `10` | +| `syncLimit` | How far out of date a server can be from a leader | `5` | +| `maxClientCnxns` | Number of concurrent connections that a single client may make to a single member | `60` | +| `maxSessionTimeout` | Maximum session timeout in milliseconds that the server will allow the client to negotiate. | `40000` | +| `autopurge.snapRetainCount` | Number of retains snapshots for autopurge | `3` | +| `autopurge.purgeInterval` | The time interval in hours for which the purge task has to be triggered | `0` | +| `fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr, mntr` | +| `listenOnAllIPs` | Allow Zookeeper to listen for connections from its peers on all available IP addresses. | `false` | +| `allowAnonymousLogin` | Allow to accept connections from unauthenticated users | `yes` | +| `auth.existingSecret` | Use existing secret (ignores previous password) | `nil` | +| `auth.enabled` | Enable ZooKeeper auth | `false` | +| `auth.clientUser` | User that will use ZooKeeper clients to auth | `nil` | +| `auth.clientPassword` | Password that will use ZooKeeper clients to auth | `nil` | +| `auth.serverUsers` | List of user to be created | `nil` | +| `auth.serverPasswords` | List of passwords to assign to users when created | `nil` | +| `heapSize` | Size in MB for the Java Heap options (Xmx and XMs) | `[]` | +| `logLevel` | Log level of ZooKeeper server | `ERROR` | +| `jvmFlags` | Default JVMFLAGS for the ZooKeeper process | `nil` | +| `config` | Configure ZooKeeper with a custom zoo.conf file | `nil` | +| `dataLogDir` | Data log directory | `""` | + +### Statefulset parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | Number of ZooKeeper nodes | `1` | +| `updateStrategy` | Update strategy for the statefulset | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `podManagementPolicy` | Pod management policy | `Parallel` | +| `podLabels` | ZooKeeper pod labels | `{}` (evaluated as a template) | +| `podAnnotations` | ZooKeeper Pod annotations | `{}` (evaluated as a template) | +| `affinity` | Affinity for pod assignment | `{}` (evaluated as a template) | +| `nodeSelector` | Node labels for pod assignment | `{}` (evaluated as a template) | +| `tolerations` | Tolerations for pod assignment | `[]` (evaluated as a template) | +| `priorityClassName` | Name of the existing priority class to be used by ZooKeeper pods | `""` | +| `securityContext.enabled` | Enable security context (ZooKeeper master pod) | `true` | +| `securityContext.fsGroup` | Group ID for the container (ZooKeeper master pod) | `1001` | +| `securityContext.runAsUser` | User ID for the container (ZooKeeper master pod) | `1001` | +| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `livenessProbe` | Liveness probe configuration for ZooKeeper | Check `values.yaml` file | +| `readinessProbe` | Readiness probe configuration for ZooKeeper | Check `values.yaml` file | +| `extraVolumes` | Extra volumes | `nil` | +| `extraVolumeMounts` | Mount extra volume(s) | `nil` | +| `podDisruptionBudget.maxUnavailable` | Max number of pods down simultaneously | `1` | + +### Exposure parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | ZooKeeper port | `2181` | +| `service.followerPort` | ZooKeeper follower port | `2888` | +| `service.electionPort` | ZooKeeper election port | `3888` | +| `service.publishNotReadyAddresses` | If the ZooKeeper headless service should publish DNS records for not ready pods | `true` | +| `serviceAccount.create` | Enable creation of ServiceAccount for zookeeper pod | `false` | +| `serviceAccount.name` | The name of the service account to use. If not set and `create` is `true`, a name is generated | Generated using the `zookeeper.fullname` template | +| `service.tls.client_enable` | Enable tls for client connections | `false` | +| `service.tls.quorum_enable` | Enable tls for quorum protocol | `false` | +| `service.tls.disable_base_client_port` | Remove client port from service definitions. | `false` | +| `service.tls.client_port` | Service port for tls client connections | `3181` | +| `service.tls.client_keystore_path` | KeyStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_key_store/key_store_file` | +| `service.tls.client_keystore_password` | KeyStore password. You can use environment variables. | `nil` | +| `service.tls.client_truststore_path` | TrustStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_trust_store/trust_store_file` | +| `service.tls.client_truststore_password` | TrustStore password. You can use environment variables. | `nil` | +| `service.tls.quorum_keystore_path` | KeyStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_key_store/key_store_file` | +| `service.tls.quorum_keystore_password` | KeyStore password. You can use environment variables. | `nil` | +| `service.tls.quorum_truststore_path` | TrustStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_trust_store/trust_store_file` | +| `service.tls.quorum_truststore_password` | TrustStore password. You can use environment variables. | `nil` | +| `service.annotations` | Annotations for the Service | `{}` | +| `service.headless.annotations` | Annotations for the Headless Service | `{}` | +| `networkPolicy.enabled` | Enable NetworkPolicy | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | + +### Persistence parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `persistence.enabled` | Enable Zookeeper data persistence using PVC | `true` | +| `persistence.existingClaim` | Provide an existing `PersistentVolumeClaim` | `nil` (evaluated as a template) | +| `persistence.storageClass` | PVC Storage Class for ZooKeeper data volume | `nil` | +| `persistence.accessMode` | PVC Access Mode for ZooKeeper data volume | `ReadWriteOnce` | +| `persistence.size` | PVC Storage Request for ZooKeeper data volume | `8Gi` | +| `persistence.annotations` | Annotations for the PVC | `{}` (evaluated as a template) | +| `persistence.dataLogDir.size` | PVC Storage Request for ZooKeeper's Data log directory | `8Gi` | +| `persistence.dataLogDir.existingClaim` | Provide an existing `PersistentVolumeClaim` for Zookeeper's Data log directory | `nil` (evaluated as a template) | + +### Volume Permissions parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `volumePermissions.enabled` | Enable init container that changes the owner and group of the persistent volume(s) mountpoint to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/minideb` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `buster` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.resources` | Init container resource requests/limit | `nil` | + +### Metrics parameters + +| Parameter | Description | Default | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------| +| `metrics.enabled` | Enable prometheus to access zookeeper metrics endpoint | `false` | +| `metrics.containerPort` | Port where a Jetty server will expose Prometheus metrics | `9141` | +| `metrics.service.type` | Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for Jetty server exposing Prometheus metrics | `ClusterIP` | +| `metrics.service.port` | Prometheus metrics service port | `9141` | +| `metrics.service.annotations` | Service annotations for Prometheus to auto-discover the metrics endpoint | `{prometheus.io/scrape: "true", prometheus.io/port: "9141"}` | +| `metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (also requires `metrics.enabled` to be `true`) | `false` | +| `metrics.serviceMonitor.namespace` | Namespace for the ServiceMonitor Resource | The Release Namespace | +| `metrics.serviceMonitor.interval` | Interval at which metrics should be scraped. | `nil` (Prometheus Operator default value) | +| `metrics.serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `nil` (Prometheus Operator default value) | +| `metrics.serviceMonitor.selector` | Prometheus instance selector labels | `nil` | +| `metrics.prometheusRule.enabled` | if `true`, creates a Prometheus Operator PrometheusRule (also requires `metrics.enabled` to be `true` and `metrics.prometheusRule.rules`) | `false` | +| `metrics.prometheusRule.namespace` | Namespace for the PrometheusRule Resource | The Release Namespace | +| `metrics.prometheusRule.selector` | Prometheus instance selector labels | `nil` | +| `metrics.prometheusRule.rules` | Prometheus Rule definitions (see values.yaml for examples) | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install my-release \ + --set auth.clientUser=newUser \ + bitnami/zookeeper +``` + +The above command sets the ZooKeeper user to `newUser`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +$ helm install my-release -f values.yaml bitnami/zookeeper +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Production configuration + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one. + +- Number of ZooKeeper nodes: + +```diff +- replicaCount: 1 ++ replicaCount: 3 +``` + +- Enable prometheus metrics: + +```diff +- metrics.enabled: false ++ metrics.enabled: true +``` + +### Log level + +You can configure the ZooKeeper log level using the `ZOO_LOG_LEVEL` environment variable. By default, it is set to `ERROR` because of each readiness probe produce an `INFO` message on connection and a `WARN` message on disconnection. + +## Persistence + +The [Bitnami ZooKeeper](https://github.com/bitnami/bitnami-docker-zookeeper) image stores the ZooKeeper data and configurations at the `/bitnami/zookeeper` path of the container. + +Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. +See the [Parameters](#parameters) section to configure the PVC or to disable persistence. + +### Adjust permissions of persistent volume mountpoint + +As the image run as non-root by default, it is necessary to adjust the ownership of the persistent volume so that the container can write data into it. + +By default, the chart is configured to use Kubernetes Security Context to automatically change the ownership of the volume. However, this feature does not work in all Kubernetes distributions. +As an alternative, this chart supports using an initContainer to change the ownership of the volume before mounting it in the final destination. + +You can enable this initContainer by setting `volumePermissions.enabled` to `true`. + +### Data Log Directory + +You can use a dedicated device for logs (instead of using the data directory) to help avoiding competition between logging and snapshots. To do so, set the `dataLogDir` parameter with the path to be used for writing transaction logs. Alternatively, set this parameter with an empty string an it result in the log being written to the data directory (Zookeeper's default behavior). + +When using a dedicated device for logs, you can use a PVC to persist the logs. To do so, set `persistence.enabled` to `true`. See the [Persistence Parameters](#persistence-parameters) section for more information. + +## Upgrading + +### To 5.21.0 + +A couple of parameters related to Zookeeper metrics were renamed or disappeared in favor of new ones: + +- `metrics.port` is renamed to `metrics.containerPort`. +- `metrics.annotations` is deprecated in favor of `metrics.service.annotations`. + +### To 3.0.0 + +This new version of the chart includes the new ZooKeeper major version 3.5.5. Note that to perform an automatic upgrade +of the application, each node will need to have at least one snapshot file created in the data directory. If not, the +new version of the application won't be able to start the service. Please refer to [ZOOKEEPER-3056](https://issues.apache.org/jira/browse/ZOOKEEPER-3056) +in order to find ways to workaround this issue in case you are facing it. + +### To 2.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's statefulsets. +Use the workaround below to upgrade from versions previous to 2.0.0. The following example assumes that the release name is `zookeeper`: + +```console +$ kubectl delete statefulset zookeeper-zookeeper --cascade=false +``` + +### To 1.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's deployments. +Use the workaround below to upgrade from versions previous to 1.0.0. The following example assumes that the release name is zookeeper: + +```console +$ kubectl delete statefulset zookeeper-zookeeper --cascade=false +``` diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt new file mode 100755 index 000000000..3cc2edbed --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt @@ -0,0 +1,57 @@ +{{- if contains .Values.service.type "LoadBalancer" }} +{{- if not .Values.auth.clientPassword }} +------------------------------------------------------------------------------- + WARNING + + By specifying "serviceType=LoadBalancer" and not specifying "auth.enabled=true" + you have most likely exposed the ZooKeeper service externally without any + authentication mechanism. + + For security reasons, we strongly suggest that you switch to "ClusterIP" or + "NodePort". As alternative, you can also specify a valid password on the + "auth.clientPassword" parameter. + +------------------------------------------------------------------------------- +{{- end }} +{{- end }} + +** Please be patient while the chart is being deployed ** + +ZooKeeper can be accessed via port 2181 on the following DNS name from within your cluster: + + {{ template "zookeeper.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + +To connect to your ZooKeeper server run the following commands: + + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "zookeeper.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].metadata.name}") + kubectl exec -it $POD_NAME -- zkCli.sh + +To connect to your ZooKeeper server from outside the cluster execute the following commands: + +{{- if contains "NodePort" .Values.service.type }} + + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "zookeeper.fullname" . }}) + zkCli.sh $NODE_IP:$NODE_PORT + +{{- else if contains "LoadBalancer" .Values.service.type }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "zookeeper.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "zookeeper.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + zkCli.sh $SERVICE_IP:2181 + +{{- else if contains "ClusterIP" .Values.service.type }} + + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "zookeeper.fullname" . }} 2181:2181 & + zkCli.sh 127.0.0.1:2181 + +{{- end }} + +{{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }} + +WARNING: Rolling tag detected ({{ .Values.image.repository }}:{{ .Values.image.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ + +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl new file mode 100755 index 000000000..f82502d69 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl @@ -0,0 +1,212 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "zookeeper.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "zookeeper.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "zookeeper.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + Create the name of the service account to use + */}} +{{- define "zookeeper.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "zookeeper.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Zookeeper image name +*/}} +{{- define "zookeeper.image" -}} +{{- $registryName := .Values.image.registry -}} +{{- $repositoryName := .Values.image.repository -}} +{{- $tag := .Values.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "zookeeper.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if or .Values.image.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- else if or .Values.image.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "zookeeper.labels" -}} +app.kubernetes.io/name: {{ include "zookeeper.name" . }} +helm.sh/chart: {{ include "zookeeper.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "zookeeper.tplValue" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "zookeeper.tplValue" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "zookeeper.matchLabels" -}} +app.kubernetes.io/name: {{ include "zookeeper.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Return ZooKeeper Client Password +*/}} +{{- define "zookeeper.clientPassword" -}} +{{- if .Values.auth.clientPassword -}} + {{- .Values.auth.clientPassword -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return ZooKeeper Servers Passwords +*/}} +{{- define "zookeeper.serverPasswords" -}} +{{- if .Values.auth.serverPasswords -}} + {{- .Values.auth.serverPasswords -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "zookeeper.volumePermissions.image" -}} +{{- $registryName := .Values.volumePermissions.image.registry -}} +{{- $repositoryName := .Values.volumePermissions.image.repository -}} +{{- $tag := .Values.volumePermissions.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Storage Class +*/}} +{{- define "zookeeper.storageClass" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +*/}} +{{- if .Values.global -}} + {{- if .Values.global.storageClass -}} + {{- if (eq "-" .Values.global.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.global.storageClass -}} + {{- end -}} + {{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml new file mode 100755 index 000000000..1a4061565 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.config }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + zoo.cfg: |- +{{ .Values.config | indent 4 }} +{{- end -}} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml new file mode 100755 index 000000000..3e26ed6c8 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml @@ -0,0 +1,29 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "zookeeper.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.metrics.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.metrics.service.annotations }} + {{ include "zookeeper.tplValue" ( dict "value" .Values.metrics.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type }} + ports: + - name: tcp-metrics + port: {{ .Values.metrics.service.port }} + targetPort: metrics + selector: {{- include "zookeeper.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml new file mode 100755 index 000000000..f7e30b4bc --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ include "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + ingress: + # Allow inbound connections to zookeeper + - ports: + - port: {{ .Values.service.port }} + from: + {{- if not .Values.networkPolicy.allowExternal }} + - podSelector: + matchLabels: + {{ include "zookeeper.fullname" . }}-client: "true" + - podSelector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 14 }} + {{- else }} + - podSelector: + matchLabels: {} + {{- end }} + # Internal ports + - ports: &intranodes_ports + - port: {{ .Values.service.followerPort }} + - port: {{ .Values.service.electionPort }} + from: + - podSelector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 14 }} + egress: + - ports: *intranodes_ports + # Allow outbound connections from zookeeper nodes + +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..818950c66 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml @@ -0,0 +1,21 @@ +{{- $replicaCount := int .Values.replicaCount }} +{{- if gt $replicaCount 1 }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: zookeeper + {{- toYaml .Values.podDisruptionBudget | nindent 2 }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml new file mode 100755 index 000000000..9cda3985c --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled .Values.metrics.prometheusRule.rules }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "zookeeper.fullname" . }} + {{- if .Values.metrics.prometheusRule.namespace }} + namespace: {{ .Values.metrics.prometheusRule.namespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- range $key, $value := .Values.metrics.prometheusRule.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + groups: + - name: {{ include "zookeeper.fullname" . }} + rules: {{- toYaml .Values.metrics.prometheusRule.rules | nindent 6 }} +{{- end }} + diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml new file mode 100755 index 000000000..b3d727fec --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.auth.enabled (not .Values.auth.existingSecret) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + client-password: {{ include "zookeeper.clientPassword" . | b64enc | quote }} + server-password: {{ include "zookeeper.serverPasswords" . | b64enc | quote }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml new file mode 100755 index 000000000..3f7ef39fd --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "zookeeper.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + role: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml new file mode 100755 index 000000000..5782dad59 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "zookeeper.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: zookeeper + endpoints: + - port: tcp-metrics + path: "/metrics" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml new file mode 100755 index 000000000..fa1e5231f --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml @@ -0,0 +1,334 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + role: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + serviceName: {{ template "zookeeper.fullname" . }}-headless + replicas: {{ .Values.replicaCount }} + podManagementPolicy: {{ .Values.podManagementPolicy }} + updateStrategy: + type: {{ .Values.updateStrategy }} + {{- if (eq "Recreate" .Values.updateStrategy) }} + rollingUpdate: null + {{- else if .Values.rollingUpdatePartition }} + rollingUpdate: + partition: {{ .Values.rollingUpdatePartition }} + {{- end }} + selector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: zookeeper + template: + metadata: + name: {{ template "zookeeper.fullname" . }} + labels: {{- include "zookeeper.labels" . | nindent 8 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.podLabels }} + {{- include "zookeeper.tplValue" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + {{- include "zookeeper.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ template "zookeeper.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "zookeeper.tplValue" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "zookeeper.tplValue" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "zookeeper.tplValue" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }} + initContainers: + - name: volume-permissions + image: {{ template "zookeeper.volumePermissions.image" . }} + imagePullPolicy: {{ default "" .Values.volumePermissions.image.pullPolicy | quote }} + command: + - chown + args: + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} + - /bitnami/zookeeper + {{- if .Values.dataLogDir }} + - {{ .Values.dataLogDir }} + {{- end }} + securityContext: + runAsUser: 0 + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + {{- if .Values.dataLogDir }} + - name: data-log + mountPath: {{ .Values.dataLogDir }} + {{- end }} + {{- end }} + containers: + - name: zookeeper + image: {{ template "zookeeper.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + command: + - bash + - -ec + - | + # Execute entrypoint as usual after obtaining ZOO_SERVER_ID based on POD hostname + HOSTNAME=`hostname -s` + if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then + ORD=${BASH_REMATCH[2]} + export ZOO_SERVER_ID=$((ORD+1)) + else + echo "Failed to get index from hostname $HOST" + exit 1 + fi + exec /entrypoint.sh /run.sh + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + env: + - name: ZOO_DATA_LOG_DIR + value: {{ .Values.dataLogDir | quote }} + - name: ZOO_PORT_NUMBER + value: {{ .Values.service.port | quote }} + - name: ZOO_TICK_TIME + value: {{ .Values.tickTime | quote }} + - name: ZOO_INIT_LIMIT + value: {{ .Values.initLimit | quote }} + - name: ZOO_SYNC_LIMIT + value: {{ .Values.syncLimit | quote }} + - name: ZOO_MAX_CLIENT_CNXNS + value: {{ .Values.maxClientCnxns | quote }} + - name: ZOO_4LW_COMMANDS_WHITELIST + value: {{ .Values.fourlwCommandsWhitelist | quote }} + - name: ZOO_LISTEN_ALLIPS_ENABLED + value: {{ ternary "yes" "no" .Values.listenOnAllIPs | quote }} + - name: ZOO_AUTOPURGE_INTERVAL + value: {{ .Values.autopurge.purgeInterval | quote }} + - name: ZOO_AUTOPURGE_RETAIN_COUNT + value: {{ .Values.autopurge.snapRetainCount | quote }} + - name: ZOO_MAX_SESSION_TIMEOUT + value: {{ .Values.maxSessionTimeout | quote }} + - name: ZOO_SERVERS + {{- $replicaCount := int .Values.replicaCount }} + {{- $followerPort := int .Values.service.followerPort }} + {{- $electionPort := int .Values.service.electionPort }} + {{- $releaseNamespace := .Release.Namespace }} + {{- $zookeeperFullname := include "zookeeper.fullname" . }} + {{- $zookeeperHeadlessServiceName := printf "%s-%s" $zookeeperFullname "headless" | trunc 63 }} + {{- $clusterDomain := .Values.clusterDomain }} + value: {{ range $i, $e := until $replicaCount }}{{ $zookeeperFullname }}-{{ $e }}.{{ $zookeeperHeadlessServiceName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $followerPort }}:{{ $electionPort }} {{ end }} + - name: ZOO_ENABLE_AUTH + value: {{ ternary "yes" "no" .Values.auth.enabled | quote }} + {{- if .Values.auth.enabled }} + - name: ZOO_CLIENT_USER + value: {{ .Values.auth.clientUser | quote }} + - name: ZOO_CLIENT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.auth.existingSecret }}{{ .Values.auth.existingSecret }}{{ else }}{{ template "zookeeper.fullname" . }}{{ end }} + key: client-password + - name: ZOO_SERVER_USERS + value: {{ .Values.auth.serverUsers | quote }} + - name: ZOO_SERVER_PASSWORDS + valueFrom: + secretKeyRef: + name: {{ if .Values.auth.existingSecret }}{{ .Values.auth.existingSecret }}{{ else }}{{ template "zookeeper.fullname" . }}{{ end }} + key: server-password + {{- end }} + - name: ZOO_HEAP_SIZE + value: {{ .Values.heapSize | quote }} + - name: ZOO_LOG_LEVEL + value: {{ .Values.logLevel | quote }} + - name: ALLOW_ANONYMOUS_LOGIN + value: {{ ternary "yes" "no" .Values.allowAnonymousLogin | quote }} + {{- if .Values.jvmFlags }} + - name: JVMFLAGS + value: {{ .Values.jvmFlags | quote }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: ZOO_ENABLE_PROMETHEUS_METRICS + value: "yes" + - name: ZOO_PROMETHEUS_METRICS_PORT_NUMBER + value: {{ .Values.metrics.containerPort | quote }} + {{- end }} + {{- if .Values.service.tls.client_enable }} + - name: ZOO_TLS_CLIENT_ENABLE + value: {{ .Values.service.tls.client_enable | quote }} + - name: ZOO_TLS_CLIENT_KEYSTORE_FILE + value: {{ .Values.service.tls.client_keystore_path | quote }} + - name: ZOO_TLS_CLIENT_KEYSTORE_PASSWORD + value: {{ .Values.service.tls.client_keystore_password | quote }} + - name: ZOO_TLS_CLIENT_TRUSTSTORE_FILE + value: {{ .Values.service.tls.client_truststore_path | quote }} + - name: ZOO_TLS_CLIENT_TRUSTSTORE_PASSWORD + value: {{ .Values.service.tls.client_truststore_password | quote }} + {{ end }} + {{- if .Values.service.tls.quorum_enable }} + - name: ZOO_TLS_QUORUM_ENABLE + value: {{ .Values.service.tls.quorum_enable | quote }} + - name: ZOO_TLS_QUORUM_KEYSTORE_FILE + value: {{ .Values.service.tls.quorum_keystore_path | quote }} + - name: ZOO_TLS_QUORUM_KEYSTORE_PASSWORD + value: {{ .Values.service.tls.quorum_keystore_password | quote }} + - name: ZOO_TLS_QUORUM_TRUSTSTORE_FILE + value: {{ .Values.service.tls.quorum_truststore_path | quote }} + - name: ZOO_TLS_QUORUM_TRUSTSTORE_PASSWORD + value: {{ .Values.service.tls.quorum_truststore_password | quote }} + {{ end }} + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + {{- if .Values.extraEnvVars }} + {{- toYaml .Values.extraEnvVars | nindent 12 }} + {{- end }} + ports: + {{ if not .Values.service.tls.disable_base_client_port }} + - name: client + containerPort: {{ .Values.service.port }} + {{ end }} + {{ if .Values.service.tls.client_enable }} + - name: client-tls + containerPort: {{ .Values.service.tls.client_port }} + {{ end }} + - name: follower + containerPort: {{ .Values.service.followerPort }} + - name: election + containerPort: {{ .Values.service.electionPort }} + {{- if .Values.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.metrics.containerPort }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + {{- if not .Values.service.tls.disable_base_client_port }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.livenessProbe.probeCommandTimeout }} nc -w {{ .Values.livenessProbe.probeCommandTimeout }} localhost {{ .Values.service.port }} | grep imok'] + {{- else }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.livenessProbe.probeCommandTimeout }} openssl s_client -quiet -crlf -connect localhost:{{ .Values.service.tls.client_port }} | grep imok'] + {{- end }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + {{- if not .Values.service.tls.disable_base_client_port }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.readinessProbe.probeCommandTimeout }} nc -w {{ .Values.readinessProbe.probeCommandTimeout }} localhost {{ .Values.service.port }} | grep imok'] + {{- else }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.readinessProbe.probeCommandTimeout }} openssl s_client -quiet -crlf -connect localhost:{{ .Values.service.tls.client_port }} | grep imok'] + {{- end }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + {{- if .Values.dataLogDir }} + - name: data-log + mountPath: {{ .Values.dataLogDir }} + {{- end }} + {{- if .Values.config }} + - name: config + mountPath: /opt/bitnami/zookeeper/conf/zoo.cfg + subPath: zoo.cfg + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.config }} + - name: config + configMap: + name: {{ template "zookeeper.fullname" . }} + {{- end }} + {{- if and .Values.persistence.enabled .Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.persistence.existingClaim .) }} + {{- else if not .Values.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- if and .Values.persistence.enabled .Values.persistence.dataLogDir.existingClaim }} + - name: data-log + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.persistence.dataLogDir.existingClaim .) }} + {{- else if and ( not .Values.persistence.enabled ) .Values.dataLogDir }} + - name: data-log + emptyDir: {} + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if and .Values.persistence.enabled (not (and .Values.persistence.existingClaim .Values.persistence.dataLogDir.existingClaim) )}} + volumeClaimTemplates: + {{- if not .Values.persistence.existingClaim }} + - metadata: + name: data + annotations: + {{- range $key, $value := .Values.persistence.annotations }} + {{ $key }}: {{ $value }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- include "zookeeper.storageClass" . | nindent 8 }} + {{- end }} + {{- if and (not .Values.persistence.dataLogDir.existingClaim) .Values.dataLogDir }} + - metadata: + name: data-log + annotations: + {{- range $key, $value := .Values.persistence.annotations }} + {{ $key }}: {{ $value }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.dataLogDir.size | quote }} + {{- include "zookeeper.storageClass" . | nindent 8 }} + {{- end }} + {{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml new file mode 100755 index 000000000..972efb51d --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "zookeeper.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.commonAnnotations .Values.service.annotations }} + annotations: + {{- if .Values.service.headless.annotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.service.headless.annotations "context" $ ) | nindent 4 }}\ + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: {{ .Values.service.publishNotReadyAddresses }} + ports: + {{ if not .Values.service.tls.disable_base_client_port }} + - name: tcp-client + port: 2181 + targetPort: client + {{ end }} + {{ if .Values.service.tls.client_enable }} + - name: tcp-client-tls + port: {{ .Values.service.tls.client_port }} + targetPort: client-tls + {{ end }} + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: {{- include "zookeeper.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml new file mode 100755 index 000000000..da3a2895a --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.commonAnnotations .Values.service.annotations }} + annotations: + {{- if .Values.service.annotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.service.annotations "context" $ ) | nindent 4 }}\ + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + {{ if not .Values.service.tls.disable_base_client_port }} + - name: tcp-client + port: 2181 + targetPort: client + {{ end }} + {{ if .Values.service.tls.client_enable }} + - name: tcp-client-tls + port: {{ .Values.service.tls.client_port }} + targetPort: client-tls + {{ end }} + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: {{- include "zookeeper.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml new file mode 100755 index 000000000..7d678603f --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml @@ -0,0 +1,430 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Zookeeper image version +## ref: https://hub.docker.com/r/bitnami/zookeeper/tags/ +## +image: + registry: docker.io + repository: bitnami/zookeeper + tag: 3.6.2-debian-10-r10 + + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + ## + debug: false + +## String to partially override zookeeper.fullname template (will maintain the release name) +# nameOverride: + +## String to fully override zookeeper.fullname template +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Example Use Cases: +## mount certificates to enable tls +# extraVolumes: +# - name: zookeeper-keystore +# secret: +# defaultMode: 288 +# secretName: zookeeper-keystore +# - name: zookeeper-trustsore +# secret: +# defaultMode: 288 +# secretName: zookeeper-truststore +# extraVolumeMounts: +# - name: zookeeper-keystore +# mountPath: /certs/keystore +# readOnly: true +# - name: zookeeper-truststore +# mountPath: /certs/truststore +# readOnly: true + + +## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Limits the number of pods of the replicated application that are down simultaneously from voluntary disruptions +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +podDisruptionBudget: + maxUnavailable: 1 + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy +## +podManagementPolicy: Parallel + +## Number of ZooKeeper nodes +## +replicaCount: 3 + +## Basic time unit in milliseconds used by ZooKeeper for heartbeats +## +tickTime: 2000 + +## ZooKeeper uses to limit the length of time the ZooKeeper servers in quorum have to connect to a leader +## +initLimit: 10 + +## How far out of date a server can be from a leader +## +syncLimit: 5 + +## Limits the number of concurrent connections that a single client may make to a single member of the ZooKeeper ensemble +## +maxClientCnxns: 60 + +## Maximum session timeout in milliseconds that the server will allow the client to negotiate. Defaults to 20 times the tickTime. +## +maxSessionTimeout: 40000 + +## A list of comma separated Four Letter Words commands to use +## +fourlwCommandsWhitelist: srvr, mntr, ruok + +## Allow zookeeper to listen for peers on all IPs +## +listenOnAllIPs: false + +## Allow to accept connections from unauthenticated users +## +allowAnonymousLogin: true + +autopurge: + ## Retains the snapRetainCount most recent snapshots and the corresponding transaction logs and deletes the rest + ## + snapRetainCount: 3 + ## The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. + ## + purgeInterval: 0 + +auth: + ## Use existing secret (ignores previous password) + ## + # existingSecret: + ## Enable Zookeeper auth. It uses SASL/Digest-MD5 + ## + enabled: false + ## User that will use Zookeeper clients to auth + ## + clientUser: + ## Password that will use Zookeeper clients to auth + ## + clientPassword: + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + serverUsers: + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + serverPasswords: + +## Size in MB for the Java Heap options (Xmx and XMs). This env var is ignored if Xmx an Xms are configured via JVMFLAGS +## +heapSize: 1024 + +## Log level for the Zookeeper server. ERROR by default. Have in mind if you set it to INFO or WARN the ReadinessProve will produce a lot of logs. +## +logLevel: ERROR + +## Data log directory. Specifying this option will direct zookeeper to write the transaction log to the dataLogDir rather than the dataDir. +## This allows a dedicated log device to be used, and helps avoid competition between logging and snaphots. +## Example: +## dataLogDir: /bitnami/zookeeper/dataLog +## +dataLogDir: "" + +## Default JVMFLAGS for the ZooKeeper process +## +# jvmFlags: + +## Configure ZooKeeper with a custom zoo.cfg file +## +# config: + +## Kubernetes configuration +## For minikube, set this to NodePort, elsewhere use LoadBalancer +## +service: + type: ClusterIP + port: 2181 + followerPort: 2888 + electionPort: 3888 + publishNotReadyAddresses: true + tls: + client_enable: true + quorum_enable: true + disable_base_client_port: true + + client_port: 3181 + + client_keystore_path: /tls_key_store/key_store_file + client_keystore_password: "" + client_truststore_path: /tls_trust_store/trust_store_file + client_truststore_password: "" + + quorum_keystore_path: /tls_key_store/key_store_file + quorum_keystore_password: "" + quorum_truststore_path: /tls_trust_store/trust_store_file + quorum_truststore_password: "" + annotations: {} + headless: + annotations: {} + +## Service account for Zookeeper to use. +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the zookeeper.fullname template + # name: + +## Zookeeper Pod Security Context +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Zookeeper data Persistent Volume Storage Class +## If defined, storageClassName: +## If set to "-", storageClassName: "", which disables dynamic provisioning +## If undefined (the default) or set to null, no storageClassName spec is +## set, choosing the default provisioner. (gp2 on AWS, standard on +## GKE, AWS & OpenStack) +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + dataLogDir: + size: 8Gi + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Labels +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +## Annotations +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +## Name of the priority class to be used by zookeeper pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: "" + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Scheduler name +## https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: stork + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 256Mi + cpu: 250m + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +## Network policies +## Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ +## +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: true + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis is listening + ## on. When true, zookeeper accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + +## Zookeeper Prometheus Exporter configuration +## +metrics: + enabled: false + + ## Zookeeper Prometheus Exporter container port + ## + containerPort: 9141 + + ## Service configuration + ## + service: + ## Zookeeper Prometheus Exporter service type + ## + type: ClusterIP + ## Zookeeper Prometheus Exporter service port + ## + port: 9141 + ## Annotations for the Zookeeper Prometheus Exporter metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace for the ServiceMonitor Resource (defaults to the Release Namespace) + ## + namespace: + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Prometheus Operator PrometheusRule configuration + ## + prometheusRule: + enabled: false + ## Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: + + ## PrometheusRule selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Some example rules. + rules: [] + # - alert: ZookeeperSyncedFollowers + # annotations: + # message: The number of synced followers for the leader node in Zookeeper deployment my-release is less than 2. This usually means that some of the Zookeeper nodes aren't communicating properly. If it doesn't resolve itself you can try killing the pods (one by one). + # expr: max(synced_followers{service="my-release-metrics"}) < 2 + # for: 5m + # labels: + # severity: critical + # - alert: ZookeeperOutstandingRequests + # annotations: + # message: The number of outstanding requests for Zookeeper pod {{ $labels.pod }} is greater than 10. This can indicate a performance issue with the Pod or cluster a whole. + # expr: outstanding_requests{service="my-release-metrics"} > 10 + # for: 5m + # labels: + # severity: critical diff --git a/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml new file mode 100755 index 000000000..a40decb54 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml @@ -0,0 +1,430 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Zookeeper image version +## ref: https://hub.docker.com/r/bitnami/zookeeper/tags/ +## +image: + registry: docker.io + repository: bitnami/zookeeper + tag: 3.6.2-debian-10-r10 + + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + ## + debug: false + +## String to partially override zookeeper.fullname template (will maintain the release name) +# nameOverride: + +## String to fully override zookeeper.fullname template +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Example Use Cases: +## mount certificates to enable tls +# extraVolumes: +# - name: zookeeper-keystore +# secret: +# defaultMode: 288 +# secretName: zookeeper-keystore +# - name: zookeeper-trustsore +# secret: +# defaultMode: 288 +# secretName: zookeeper-truststore +# extraVolumeMounts: +# - name: zookeeper-keystore +# mountPath: /certs/keystore +# readOnly: true +# - name: zookeeper-truststore +# mountPath: /certs/truststore +# readOnly: true + +## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Limits the number of pods of the replicated application that are down simultaneously from voluntary disruptions +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +podDisruptionBudget: + maxUnavailable: 1 + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy +## +podManagementPolicy: Parallel + +## Number of ZooKeeper nodes +## +replicaCount: 1 + +## Basic time unit in milliseconds used by ZooKeeper for heartbeats +## +tickTime: 2000 + +## ZooKeeper uses to limit the length of time the ZooKeeper servers in quorum have to connect to a leader +## +initLimit: 10 + +## How far out of date a server can be from a leader +## +syncLimit: 5 + +## Limits the number of concurrent connections that a single client may make to a single member of the ZooKeeper ensemble +## +maxClientCnxns: 60 + +## A list of comma separated Four Letter Words commands to use +## +fourlwCommandsWhitelist: srvr, mntr, ruok + +## Allow zookeeper to listen for peers on all IPs +## +listenOnAllIPs: false + +## Allow to accept connections from unauthenticated users +## +allowAnonymousLogin: true + +autopurge: + ## Retains the snapRetainCount most recent snapshots and the corresponding transaction logs and deletes the rest + ## + snapRetainCount: 3 + ## The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. + ## + purgeInterval: 0 + +## Maximum session timeout in milliseconds that the server will allow the client to negotiate. Defaults to 20 times the tickTime. +## +maxSessionTimeout: 40000 + +auth: + ## Use existing secret (ignores previous password) + ## + # existingSecret: + ## Enable Zookeeper auth. It uses SASL/Digest-MD5 + ## + enabled: false + ## User that will use Zookeeper clients to auth + ## + clientUser: + ## Password that will use Zookeeper clients to auth + ## + clientPassword: + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + serverUsers: + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + serverPasswords: + +## Size in MB for the Java Heap options (Xmx and XMs). This env var is ignored if Xmx an Xms are configured via JVMFLAGS +## +heapSize: 1024 + +## Log level for the Zookeeper server. ERROR by default. Have in mind if you set it to INFO or WARN the ReadinessProve will produce a lot of logs. +## +logLevel: ERROR + +## Data log directory. Specifying this option will direct zookeeper to write the transaction log to the dataLogDir rather than the dataDir. +## This allows a dedicated log device to be used, and helps avoid competition between logging and snaphots. +## Example: +## dataLogDir: /bitnami/zookeeper/dataLog +## +dataLogDir: "" + +## Default JVMFLAGS for the ZooKeeper process +## +# jvmFlags: + +## Configure ZooKeeper with a custom zoo.cfg file +## +# config: + +## Kubernetes configuration +## For minikube, set this to NodePort, elsewhere use LoadBalancer +## +service: + type: ClusterIP + port: 2181 + followerPort: 2888 + electionPort: 3888 + publishNotReadyAddresses: true + tls: + client_enable: false + quorum_enable: false + disable_base_client_port: false + + client_port: 3181 + + client_keystore_path: /tls_key_store/key_store_file + client_keystore_password: "" + client_truststore_path: /tls_trust_store/trust_store_file + client_truststore_password: "" + + quorum_keystore_path: /tls_key_store/key_store_file + quorum_keystore_password: "" + quorum_truststore_path: /tls_trust_store/trust_store_file + quorum_truststore_password: "" + annotations: {} + headless: + annotations: {} + +## Service account for Zookeeper to use. +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the zookeeper.fullname template + # name: + +## Zookeeper Pod Security Context +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Zookeeper data Persistent Volume Storage Class +## If defined, storageClassName: +## If set to "-", storageClassName: "", which disables dynamic provisioning +## If undefined (the default) or set to null, no storageClassName spec is +## set, choosing the default provisioner. (gp2 on AWS, standard on +## GKE, AWS & OpenStack) +## +persistence: + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + + enabled: true + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + dataLogDir: + size: 8Gi + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Labels +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +## Annotations +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +## Name of the priority class to be used by zookeeper pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: "" + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Scheduler name +## https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: stork + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 256Mi + cpu: 250m + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +## Network policies +## Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ +## +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis is listening + ## on. When true, zookeeper accept connections from any source + ## (with the correct destination port). + ## + # allowExternal: true + +## Zookeeper Prometheus Exporter configuration +## +metrics: + enabled: false + + ## Zookeeper Prometheus Exporter container port + ## + containerPort: 9141 + + ## Service configuration + ## + service: + ## Zookeeper Prometheus Exporter service type + ## + type: ClusterIP + ## Zookeeper Prometheus Exporter service port + ## + port: 9141 + ## Annotations for the Zookeeper Prometheus Exporter metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace for the ServiceMonitor Resource (defaults to the Release Namespace) + ## + namespace: + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Prometheus Operator PrometheusRule configuration + ## + prometheusRule: + enabled: false + ## Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: + + ## PrometheusRule selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Some example rules. + rules: [] + # - alert: ZookeeperSyncedFollowers + # annotations: + # message: The number of synced followers for the leader node in Zookeeper deployment my-release is less than 2. This usually means that some of the Zookeeper nodes aren't communicating properly. If it doesn't resolve itself you can try killing the pods (one by one). + # expr: max(synced_followers{service="my-release-metrics"}) < 2 + # for: 5m + # labels: + # severity: critical + # - alert: ZookeeperOutstandingRequests + # annotations: + # message: The number of outstanding requests for Zookeeper pod {{ $labels.pod }} is greater than 10. This can indicate a performance issue with the Pod or cluster a whole. + # expr: outstanding_requests{service="my-release-metrics"} > 10 + # for: 5m + # labels: + # severity: critical diff --git a/ee/scripts/helm/helm/databases/charts/kafka/files/jks/README.md b/ee/scripts/helm/helm/databases/charts/kafka/files/jks/README.md new file mode 100755 index 000000000..e110a8825 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/files/jks/README.md @@ -0,0 +1,10 @@ +# Java Key Stores + +You can copy here your Java Key Stores (JKS) files so a secret is created including them. Remember to use a truststore (`kafka.truststore.jks`) and one keystore (`kafka.keystore.jks`) per Kafka broker you have in the cluster. For instance, if you have 3 brokers you need to copy here the following files: + +- kafka.truststore.jks +- kafka-0.keystore.jks +- kafka-1.keystore.jks +- kafka-2.keystore.jks + +Find more info in [this section](https://github.com/bitnami/charts/tree/master/bitnami/kafka#enable-security-for-kafka-and-zookeeper) of the README.md file. diff --git a/ee/scripts/helm/helm/databases/charts/kafka/kafka.yaml b/ee/scripts/helm/helm/databases/charts/kafka/kafka.yaml new file mode 100644 index 000000000..acd718957 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/kafka.yaml @@ -0,0 +1,521 @@ +--- +# Source: kafka/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kafka + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +--- +# Source: kafka/templates/scripts-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-scripts + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm +data: + setup.sh: |- + #!/bin/bash + + ID="${MY_POD_NAME#"kafka-"}" + export KAFKA_CFG_BROKER_ID="$ID" + + exec /entrypoint.sh /run.sh +--- +# Source: kafka/charts/zookeeper/templates/svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-zookeeper-headless + namespace: db + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + + - name: tcp-client + port: 2181 + targetPort: client + + + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: zookeeper +--- +# Source: kafka/charts/zookeeper/templates/svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-zookeeper + namespace: db + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper +spec: + type: ClusterIP + ports: + + - name: tcp-client + port: 2181 + targetPort: client + + + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: zookeeper +--- +# Source: kafka/templates/kafka-metrics-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-metrics + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + annotations: + + prometheus.io/path: /metrics + prometheus.io/port: '9308' + prometheus.io/scrape: "true" +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 9308 + protocol: TCP + targetPort: metrics + nodePort: null + selector: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: metrics +--- +# Source: kafka/templates/svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-headless + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-client + port: 9092 + protocol: TCP + targetPort: kafka-client + - name: tcp-internal + port: 9093 + protocol: TCP + targetPort: kafka-internal + selector: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: kafka +--- +# Source: kafka/templates/svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +spec: + type: ClusterIP + ports: + - name: tcp-client + port: 9092 + protocol: TCP + targetPort: kafka-client + nodePort: null + selector: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: kafka +--- +# Source: kafka/templates/kafka-metrics-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-exporter + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: metrics + template: + metadata: + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + spec: + containers: + - name: kafka-exporter + image: docker.io/bitnami/kafka-exporter:1.2.0-debian-10-r220 + imagePullPolicy: "IfNotPresent" + command: + - /bin/bash + - -ec + - | + read -r -a sasl_passwords <<< "$(tr ',;' ' ' <<< "${SASL_USER_PASSWORD}")" + kafka_exporter \ + --kafka.server=kafka-0.kafka-headless.db.svc.cluster.local:9092 \ + --kafka.server=kafka-1.kafka-headless.db.svc.cluster.local:9092 \ + --web.listen-address=:9308 + ports: + - name: metrics + containerPort: 9308 + resources: + limits: {} + requests: {} +--- +# Source: kafka/charts/zookeeper/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka-zookeeper + namespace: db + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper + role: zookeeper +spec: + serviceName: kafka-zookeeper-headless + replicas: 1 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: zookeeper + template: + metadata: + name: kafka-zookeeper + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper + spec: + + serviceAccountName: default + securityContext: + fsGroup: 1001 + containers: + - name: zookeeper + image: docker.io/bitnami/zookeeper:3.6.2-debian-10-r10 + imagePullPolicy: "IfNotPresent" + securityContext: + runAsUser: 1001 + command: + - bash + - -ec + - | + # Execute entrypoint as usual after obtaining ZOO_SERVER_ID based on POD hostname + HOSTNAME=`hostname -s` + if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then + ORD=${BASH_REMATCH[2]} + export ZOO_SERVER_ID=$((ORD+1)) + else + echo "Failed to get index from hostname $HOST" + exit 1 + fi + exec /entrypoint.sh /run.sh + resources: + requests: + cpu: 250m + memory: 256Mi + env: + - name: ZOO_DATA_LOG_DIR + value: "" + - name: ZOO_PORT_NUMBER + value: "2181" + - name: ZOO_TICK_TIME + value: "2000" + - name: ZOO_INIT_LIMIT + value: "10" + - name: ZOO_SYNC_LIMIT + value: "5" + - name: ZOO_MAX_CLIENT_CNXNS + value: "60" + - name: ZOO_4LW_COMMANDS_WHITELIST + value: "srvr, mntr, ruok" + - name: ZOO_LISTEN_ALLIPS_ENABLED + value: "no" + - name: ZOO_AUTOPURGE_INTERVAL + value: "0" + - name: ZOO_AUTOPURGE_RETAIN_COUNT + value: "3" + - name: ZOO_MAX_SESSION_TIMEOUT + value: "40000" + - name: ZOO_SERVERS + value: kafka-zookeeper-0.kafka-zookeeper-headless.db.svc.cluster.local:2888:3888 + - name: ZOO_ENABLE_AUTH + value: "no" + - name: ZOO_HEAP_SIZE + value: "1024" + - name: ZOO_LOG_LEVEL + value: "ERROR" + - name: ALLOW_ANONYMOUS_LOGIN + value: "yes" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + ports: + + - name: client + containerPort: 2181 + + + - name: follower + containerPort: 2888 + - name: election + containerPort: 3888 + livenessProbe: + exec: + command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok'] + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + exec: + command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok'] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + volumes: + volumeClaimTemplates: + - metadata: + name: data + annotations: + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" +--- +# Source: kafka/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +spec: + podManagementPolicy: Parallel + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: kafka + serviceName: kafka-headless + updateStrategy: + type: "RollingUpdate" + template: + metadata: + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka + spec: + securityContext: + fsGroup: 1001 + runAsUser: 1001 + serviceAccountName: kafka + containers: + - name: kafka + image: docker.io/bitnami/kafka:2.6.0-debian-10-r30 + imagePullPolicy: "IfNotPresent" + command: + - /scripts/setup.sh + env: + - name: BITNAMI_DEBUG + value: "false" + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KAFKA_CFG_ZOOKEEPER_CONNECT + value: "kafka-zookeeper" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: "INTERNAL" + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: "INTERNAL:PLAINTEXT,CLIENT:PLAINTEXT" + - name: KAFKA_CFG_LISTENERS + value: "INTERNAL://:9093,CLIENT://:9092" + - name: KAFKA_CFG_ADVERTISED_LISTENERS + value: "INTERNAL://$(MY_POD_NAME).kafka-headless.db.svc.cluster.local:9093,CLIENT://$(MY_POD_NAME).kafka-headless.db.svc.cluster.local:9092" + - name: ALLOW_PLAINTEXT_LISTENER + value: "yes" + - name: KAFKA_CFG_DELETE_TOPIC_ENABLE + value: "false" + - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE + value: "true" + - name: KAFKA_HEAP_OPTS + value: "-Xmx1024m -Xms1024m" + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES + value: "10000" + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS + value: "1000" + - name: KAFKA_CFG_LOG_RETENTION_BYTES + value: "1073741824" + - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS + value: "300000" + - name: KAFKA_CFG_LOG_RETENTION_HOURS + value: "168" + - name: KAFKA_CFG_MESSAGE_MAX_BYTES + value: "1000012" + - name: KAFKA_CFG_LOG_SEGMENT_BYTES + value: "1073741824" + - name: KAFKA_CFG_LOG_DIRS + value: "/bitnami/kafka/data" + - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR + value: "1" + - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: "1" + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR + value: "1" + - name: KAFKA_CFG_NUM_IO_THREADS + value: "8" + - name: KAFKA_CFG_NUM_NETWORK_THREADS + value: "3" + - name: KAFKA_CFG_NUM_PARTITIONS + value: "1" + - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR + value: "1" + - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES + value: "102400" + - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES + value: "104857600" + - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES + value: "102400" + - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS + value: "6000" + ports: + - name: kafka-client + containerPort: 9092 + - name: kafka-internal + containerPort: 9093 + livenessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 10 + timeoutSeconds: 5 + failureThreshold: + periodSeconds: + successThreshold: + readinessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + periodSeconds: + successThreshold: + resources: + limits: {} + requests: {} + volumeMounts: + - name: data + mountPath: /bitnami/kafka + - name: scripts + mountPath: /scripts/setup.sh + subPath: setup.sh + volumes: + - name: scripts + configMap: + name: kafka-scripts + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" diff --git a/ee/scripts/helm/helm/databases/charts/kafka/requirements.lock b/ee/scripts/helm/helm/databases/charts/kafka/requirements.lock new file mode 100755 index 000000000..115d0b229 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: zookeeper + repository: https://charts.bitnami.com/bitnami + version: 5.21.9 +digest: sha256:2f3c43ce02e3966648b8c89be121fe39537f62ea1d161ad908f51ddc90e4243e +generated: "2020-09-29T07:43:56.483358254Z" diff --git a/ee/scripts/helm/helm/databases/charts/kafka/requirements.yaml b/ee/scripts/helm/helm/databases/charts/kafka/requirements.yaml new file mode 100755 index 000000000..533875258 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: + - name: zookeeper + version: 5.x.x + repository: https://charts.bitnami.com/bitnami + condition: zookeeper.enabled diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt b/ee/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt new file mode 100755 index 000000000..0347c21c4 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt @@ -0,0 +1,181 @@ +{{- $replicaCount := int .Values.replicaCount -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $clusterDomain := .Values.clusterDomain -}} +{{- $fullname := include "kafka.fullname" . -}} +{{- $clientProtocol := include "kafka.listenerType" ( dict "protocol" .Values.auth.clientProtocol ) -}} +{{- $servicePort := int .Values.service.port -}} +{{- $loadBalancerIPListLength := len .Values.externalAccess.service.loadBalancerIPs -}} +{{- if and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $loadBalancerIPListLength )) (eq .Values.externalAccess.service.type "LoadBalancer") }} + +############################################################################### +### ERROR: You enabled external access to Kafka brokers without specifying ### +### the array of load balancer IPs for Kafka brokers. ### +############################################################################### + +This deployment will be incomplete until you configure the array of load balancer +IPs for Kafka brokers. To complete your deployment follow the steps below: + +1. Wait for the load balancer IPs (it may take a few minutes for them to be available): + + kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -w + +2. Obtain the load balancer IPs and upgrade your chart: + + {{- range $i, $e := until $replicaCount }} + LOAD_BALANCER_IP_{{ add $i 1 }}="$(kubectl get svc --namespace {{ $releaseNamespace }} {{ $fullname }}-{{ $i }}-external -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" + {{- end }} + +3. Upgrade you chart: + + helm upgrade {{ .Release.Name }} bitnami/{{ .Chart.Name }} \ + --set replicaCount={{ $replicaCount }} \ + --set externalAccess.enabled=true \ + {{- range $i, $e := until $replicaCount }} + --set externalAccess.service.loadBalancerIPs[{{ $i }}]=$LOAD_BALANCER_IP_{{ add $i 1 }} \ + {{- end }} + --set externalAccess.service.type=LoadBalancer + +{{- else }} + +{{- if and (or (eq .Values.service.type "LoadBalancer") .Values.externalAccess.enabled) (eq .Values.auth.clientProtocol "plaintext") }} +--------------------------------------------------------------------------------------------- + WARNING + + By specifying "serviceType=LoadBalancer" and not configuring the authentication + you have most likely exposed the Kafka service externally without any + authentication mechanism. + + For security reasons, we strongly suggest that you switch to "ClusterIP" or + "NodePort". As alternative, you can also configure the Kafka authentication. + +--------------------------------------------------------------------------------------------- +{{- end }} + +** Please be patient while the chart is being deployed ** + +Kafka can be accessed by consumers via port {{ $servicePort }} on the following DNS name from within your cluster: + + {{ $fullname }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} + +Each Kafka broker can be accessed by producers via port {{ $servicePort }} on the following DNS name(s) from within your cluster: + +{{- $brokerList := list }} +{{- range $e, $i := until $replicaCount }} +{{- $brokerList = append $brokerList (printf "%s-%d.%s-headless.%s.svc.%s:%d" $fullname $i $fullname $releaseNamespace $clusterDomain $servicePort) }} +{{- end }} +{{ join "\n" $brokerList | nindent 4 }} + + +{{- if (include "kafka.client.saslAuthentication" .) }} + +You need to configure your Kafka client to access using SASL authentication. To do so, you need to create the 'kafka_jaas.conf' and 'client.properties' configuration files by executing these commands: + + - kafka_jaas.conf: + +cat > kafka_jaas.conf < client.properties <<>(Value) + name: kafka_controller_$1_$2_$4 + labels: + broker_id: "$3" + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.controller<>(Count) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.server<>(Value) + name: kafka_server_$1_$2_$4 + labels: + client_id: "$3" + - pattern : kafka.network<>(Value) + name: kafka_network_$1_$2_$4 + labels: + network_processor: $3 + - pattern : kafka.network<>(Count) + name: kafka_network_$1_$2_$4 + labels: + request: $3 + - pattern: kafka.server<>(Count|OneMinuteRate) + name: kafka_server_$1_$2_$4 + labels: + topic: $3 + - pattern: kafka.server<>(Value) + name: kafka_server_$1_$2_$3_$4 + - pattern: kafka.server<>(Count|Value|OneMinuteRate) + name: kafka_server_$1_total_$2_$3 + - pattern: kafka.server<>(queue-size) + name: kafka_server_$1_$2 + - pattern: java.lang<(.+)>(\w+) + name: java_lang_$1_$4_$3_$2 + - pattern: java.lang<>(\w+) + name: java_lang_$1_$3_$2 + - pattern : java.lang + - pattern: kafka.log<>Value + name: kafka_log_$1_$2 + labels: + topic: $3 + partition: $4 +{{- end -}} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml new file mode 100755 index 000000000..83edd8422 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml @@ -0,0 +1,45 @@ +{{- if .Values.metrics.jmx.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }}-jmx-metrics + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.metrics.jmx.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.metrics.jmx.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" .Values.metrics.jmx.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.jmx.service.type }} + {{- if eq .Values.metrics.jmx.service.type "LoadBalancer" }} + {{- if .Values.metrics.jmx.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.metrics.jmx.service.loadBalancerIP }} + {{- end }} + {{- if .Values.metrics.jmx.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.metrics.jmx.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + {{- if and (eq .Values.metrics.jmx.service.type "ClusterIP") .Values.metrics.jmx.service.clusterIP }} + clusterIP: {{ .Values.metrics.jmx.service.clusterIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.metrics.jmx.service.port }} + protocol: TCP + targetPort: metrics + {{- if and (or (eq .Values.metrics.jmx.service.type "NodePort") (eq .Values.metrics.jmx.service.type "LoadBalancer")) (not (empty .Values.metrics.jmx.service.nodePort)) }} + nodePort: {{ .Values.metrics.jmx.service.nodePort }} + {{- else if eq .Values.metrics.jmx.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: kafka +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml new file mode 100755 index 000000000..c547fbb39 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml @@ -0,0 +1,87 @@ +{{- if .Values.metrics.kafka.enabled }} +{{- $replicaCount := int .Values.replicaCount -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $clusterDomain := .Values.clusterDomain -}} +{{- $fullname := include "kafka.fullname" . -}} +{{- $servicePort := int .Values.service.port -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kafka.fullname" . }}-exporter + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: metrics + template: + metadata: + labels: {{- include "kafka.labels" . | nindent 8 }} + app.kubernetes.io/component: metrics + spec: +{{- include "kafka.imagePullSecrets" . | indent 6 }} + containers: + - name: kafka-exporter + image: {{ include "kafka.metrics.kafka.image" . }} + imagePullPolicy: {{ .Values.metrics.kafka.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + read -r -a sasl_passwords <<< "$(tr ',;' ' ' <<< "${SASL_USER_PASSWORD}")" + kafka_exporter \ + {{- range $i, $e := until $replicaCount }} + --kafka.server={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $servicePort }} \ + {{- end }} + {{- if (include "kafka.client.saslAuthentication" .) }} + --sasl.enabled \ + --sasl.username="$SASL_USERNAME" \ + --sasl.password="${sasl_passwords[0]}" \ + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + --tls.enabled \ + {{- if .Values.metrics.kafka.certificatesSecret }} + --tls.ca-file="/opt/bitnami/kafka-exporter/certs/ca-file" \ + --tls.cert-file="/opt/bitnami/kafka-exporter/certs/cert-file" \ + --tls.key-file="/opt/bitnami/kafka-exporter/certs/key-file" \ + {{- end }} + {{- end }} + {{- range $key, $value := .Values.metrics.kafka.extraFlags }} + --{{ $key }}{{ if $value }}={{ $value }}{{ end }} \ + {{- end }} + --web.listen-address=:9308 + {{- if (include "kafka.client.saslAuthentication" .) }} + env: + - name: SASL_USERNAME + value: {{ index .Values.auth.jaas.clientUsers 0 | quote }} + - name: SASL_USER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: client-passwords + {{- end }} + ports: + - name: metrics + containerPort: 9308 + {{- if .Values.metrics.kafka.resources }} + resources: {{ toYaml .Values.metrics.kafka.resources | nindent 12 }} + {{- end }} + {{- if and (include "kafka.tlsEncryption" .) .Values.metrics.kafka.certificatesSecret }} + volumeMounts: + - name: kafka-exporter-certificates + mountPath: /opt/bitnami/kafka-exporter/certs/ + readOnly: true + volumes: + - name: kafka-exporter-certificates + secret: + secretName: {{ .Values.metrics.kafka.certificatesSecret }} + defaultMode: 0440 + {{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml new file mode 100755 index 000000000..54a4ccb0b --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml @@ -0,0 +1,45 @@ +{{- if .Values.metrics.kafka.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }}-metrics + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.metrics.kafka.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.metrics.kafka.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" .Values.metrics.kafka.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.kafka.service.type }} + {{- if eq .Values.metrics.kafka.service.type "LoadBalancer" }} + {{- if .Values.metrics.kafka.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.metrics.kafka.service.loadBalancerIP }} + {{- end }} + {{- if .Values.metrics.kafka.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.metrics.kafka.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + {{- if and (eq .Values.metrics.kafka.service.type "ClusterIP") .Values.metrics.kafka.service.clusterIP }} + clusterIP: {{ .Values.metrics.kafka.service.clusterIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.metrics.kafka.service.port }} + protocol: TCP + targetPort: metrics + {{- if and (or (eq .Values.metrics.kafka.service.type "NodePort") (eq .Values.metrics.kafka.service.type "LoadBalancer")) (not (empty .Values.metrics.kafka.service.nodePort)) }} + nodePort: {{ .Values.metrics.kafka.service.nodePort }} + {{- else if eq .Values.metrics.kafka.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: metrics +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml new file mode 100755 index 000000000..0a34d50dd --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml @@ -0,0 +1,16 @@ +{{- if (include "kafka.log4j.createConfigMap" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kafka.log4j.configMapName" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + {{ .Values.log4j | nindent 4 }} +{{- end -}} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..cf515becb --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml @@ -0,0 +1,25 @@ +{{- $replicaCount := int .Values.replicaCount }} +{{- if and .Values.pdb.create (gt $replicaCount 1) }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if .Values.pdb.minAvailable }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{- end }} + {{- if .Values.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: kafka +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/role.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/role.yaml new file mode 100755 index 000000000..943c5bf3c --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/role.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +{{- end -}} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml new file mode 100755 index 000000000..78f940f85 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.serviceAccount.create .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +roleRef: + kind: Role + name: {{ template "kafka.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ template "kafka.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml new file mode 100755 index 000000000..705545a61 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kafka.fullname" . }}-scripts + labels: {{- include "kafka.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + {{- $fullname := include "kafka.fullname" . }} + {{- $releaseNamespace := .Release.Namespace }} + {{- $clusterDomain := .Values.clusterDomain }} + {{- $interBrokerPort := .Values.service.internalPort }} + {{- $clientPort := .Values.service.port }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + auto-discovery.sh: |- + #!/bin/bash + + SVC_NAME="${MY_POD_NAME}-external" + + {{- if eq .Values.externalAccess.service.type "LoadBalancer" }} + # Auxiliar functions + retry_while() { + local -r cmd="${1:?cmd is missing}" + local -r retries="${2:-12}" + local -r sleep_time="${3:-5}" + local return_value=1 + + read -r -a command <<< "$cmd" + for ((i = 1 ; i <= retries ; i+=1 )); do + "${command[@]}" && return_value=0 && break + sleep "$sleep_time" + done + return $return_value + } + k8s_svc_lb_ip() { + local namespace=${1:?namespace is missing} + local service=${2:?service is missing} + local service_ip=$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.status.loadBalancer.ingress[0].ip}") + local service_hostname=$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") + + if [[ -n ${service_ip} ]]; then + echo "${service_ip}" + else + echo "${service_hostname}" + fi + } + k8s_svc_lb_ip_ready() { + local namespace=${1:?namespace is missing} + local service=${2:?service is missing} + [[ -n "$(k8s_svc_lb_ip "$namespace" "$service")" ]] + } + # Wait until LoadBalancer IP is ready + retry_while "k8s_svc_lb_ip_ready {{ $releaseNamespace }} $SVC_NAME" || exit 1 + # Obtain LoadBalancer external IP + k8s_svc_lb_ip "{{ $releaseNamespace }}" "$SVC_NAME" | tee "$SHARED_FILE" + {{- else if eq .Values.externalAccess.service.type "NodePort" }} + k8s_svc_node_port() { + local namespace=${1:?namespace is missing} + local service=${2:?service is missing} + local index=${3:-0} + local node_port="$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.spec.ports[$index].nodePort}")" + echo "$node_port" + } + k8s_svc_node_port "{{ $releaseNamespace }}" "$SVC_NAME" | tee "$SHARED_FILE" + {{- end }} + {{- end }} + setup.sh: |- + #!/bin/bash + + ID="${MY_POD_NAME#"{{ $fullname }}-"}" + export KAFKA_CFG_BROKER_ID="$ID" + + {{- if .Values.externalAccess.enabled }} + # Configure external ip and port + {{- if eq .Values.externalAccess.service.type "LoadBalancer" }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + export EXTERNAL_ACCESS_IP="$(<${SHARED_FILE})" + {{- else }} + export EXTERNAL_ACCESS_IP=$(echo '{{ .Values.externalAccess.service.loadBalancerIPs }}' | tr -d '[]' | cut -d ' ' -f "$(($ID + 1))") + {{- end }} + export EXTERNAL_ACCESS_PORT={{ .Values.externalAccess.service.port }} + {{- else if eq .Values.externalAccess.service.type "NodePort" }} + {{- if .Values.externalAccess.service.domain }} + export EXTERNAL_ACCESS_IP={{ .Values.externalAccess.service.domain }} + {{- else }} + export EXTERNAL_ACCESS_IP=$(curl -s https://ipinfo.io/ip) + {{- end }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + export EXTERNAL_ACCESS_PORT="$(<${SHARED_FILE})" + {{- else }} + export EXTERNAL_ACCESS_PORT=$(echo '{{ .Values.externalAccess.service.nodePorts }}' | tr -d '[]' | cut -d ' ' -f "$(($ID + 1))") + {{- end }} + {{- end }} + + # Configure Kafka advertised listeners + {{- if .Values.advertisedListeners }} + export KAFKA_CFG_ADVERTISED_LISTENERS={{ .Values.advertisedListeners }} + {{- else }} + export KAFKA_CFG_ADVERTISED_LISTENERS="INTERNAL://${MY_POD_NAME}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $interBrokerPort }},CLIENT://${MY_POD_NAME}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $clientPort }},EXTERNAL://${EXTERNAL_ACCESS_IP}:${EXTERNAL_ACCESS_PORT}" + {{- end }} + {{- end }} + + {{- if (include "kafka.tlsEncryption" .) }} + if [[ -f "/certs/kafka.truststore.jks" ]] && [[ -f "/certs/kafka-${ID}.keystore.jks" ]]; then + mkdir -p /opt/bitnami/kafka/config/certs + cp "/certs/kafka.truststore.jks" "/opt/bitnami/kafka/config/certs/kafka.truststore.jks" + cp "/certs/kafka-${ID}.keystore.jks" "/opt/bitnami/kafka/config/certs/kafka.keystore.jks" + else + echo "Couldn't find the expected Java Key Stores (JKS) files! They are mandatory when encryption via TLS is enabled." + exit 1 + fi + {{- end }} + + exec /entrypoint.sh /run.sh diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml new file mode 100755 index 000000000..790790b3f --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kafka.serviceAccountName" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml new file mode 100755 index 000000000..250bb5306 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.metrics.jmx.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kafka.fullname" . }}-jmx-metrics + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: kafka + endpoints: + - port: http-metrics + path: "/" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml new file mode 100755 index 000000000..951bf7c41 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.metrics.kafka.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kafka.fullname" . }}-metrics + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: metrics + endpoints: + - port: http-metrics + path: "/metrics" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml new file mode 100755 index 000000000..e9b5ce8f9 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml @@ -0,0 +1,435 @@ +{{- $replicaCount := int .Values.replicaCount }} +{{- $fullname := include "kafka.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} +{{- $interBrokerPort := .Values.service.internalPort }} +{{- $clientPort := .Values.service.port }} +{{- $interBrokerProtocol := include "kafka.listenerType" ( dict "protocol" .Values.auth.interBrokerProtocol ) -}} +{{- $clientProtocol := include "kafka.listenerType" ( dict "protocol" .Values.auth.clientProtocol ) -}} +{{- $loadBalancerIPListLength := len .Values.externalAccess.service.loadBalancerIPs }} +{{- if not (and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $loadBalancerIPListLength )) (eq .Values.externalAccess.service.type "LoadBalancer")) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podManagementPolicy: Parallel + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: kafka + serviceName: {{ template "kafka.fullname" . }}-headless + updateStrategy: + type: {{ .Values.updateStrategy | quote }} + {{- if (eq "OnDelete" .Values.updateStrategy) }} + rollingUpdate: null + {{- else if .Values.rollingUpdatePartition }} + rollingUpdate: + partition: {{ .Values.rollingUpdatePartition }} + {{- end }} + template: + metadata: + labels: {{- include "kafka.labels" . | nindent 8 }} + app.kubernetes.io/component: kafka + {{- if .Values.podLabels }} + {{- include "kafka.tplValue" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if or (include "kafka.createConfigmap" .) (include "kafka.createJaasSecret" .) .Values.externalAccess.enabled (include "kafka.metrics.jmx.createConfigmap" .) .Values.podAnnotations }} + annotations: + {{- if (include "kafka.createConfigmap" .) }} + checksum/configuration: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if (include "kafka.createJaasSecret" .) }} + checksum/secret: {{ include (print $.Template.BasePath "/jaas-secret.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.externalAccess.enabled }} + checksum/scripts: {{ include (print $.Template.BasePath "/scripts-configmap.yaml") . | sha256sum }} + {{- end }} + {{- if (include "kafka.metrics.jmx.createConfigmap" .) }} + checksum/jmx-configuration: {{ include (print $.Template.BasePath "/jmx-configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- include "kafka.tplValue" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + {{- end }} + spec: +{{- include "kafka.imagePullSecrets" . | indent 6 }} + {{- if .Values.affinity }} + affinity: {{- include "kafka.tplValue" ( dict "value" .Values.affinity "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "kafka.tplValue" ( dict "value" .Values.nodeSelector "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "kafka.tplValue" ( dict "value" .Values.tolerations "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- if .Values.podSecurityContext }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ template "kafka.serviceAccountName" . }} + {{- end }} + {{- if or (and .Values.volumePermissions.enabled .Values.persistence.enabled) (and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled) }} + initContainers: + {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }} + - name: volume-permissions + image: {{ include "kafka.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + args: + - -ec + - | + mkdir -p /bitnami/kafka + chown -R "{{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }}" "/bitnami/kafka" + securityContext: + runAsUser: 0 + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/kafka + {{- end }} + {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }} + - name: auto-discovery + image: {{ include "kafka.externalAccess.autoDiscovery.image" . }} + imagePullPolicy: {{ .Values.externalAccess.autoDiscovery.image.pullPolicy | quote }} + command: + - /scripts/auto-discovery.sh + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: SHARED_FILE + value: "/shared/info.txt" + {{- if .Values.externalAccess.autoDiscovery.resources }} + resources: {{- toYaml .Values.externalAccess.autoDiscovery.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: shared + mountPath: /shared + - name: scripts + mountPath: /scripts/auto-discovery.sh + subPath: auto-discovery.sh + {{- end }} + {{- end }} + containers: + - name: kafka + image: {{ include "kafka.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + command: {{- include "kafka.tplValue" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- if .Values.args }} + args: {{- include "kafka.tplValue" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" .Values.image.debug | quote }} + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KAFKA_CFG_ZOOKEEPER_CONNECT + {{- if .Values.zookeeper.enabled }} + value: {{ include "kafka.zookeeper.fullname" . | quote }} + {{- else }} + value: {{ join "," .Values.externalZookeeper.servers | quote }} + {{- end }} + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: {{ .Values.interBrokerListenerName | quote }} + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + {{- if .Values.listenerSecurityProtocolMap }} + value: {{ .Values.listenerSecurityProtocolMap | quote }} + {{- else if .Values.externalAccess.enabled }} + value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }},EXTERNAL:{{ $clientProtocol }}" + {{- else }} + value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }}" + {{- end }} + {{- if or ($clientProtocol | regexFind "SASL") ($interBrokerProtocol | regexFind "SASL") .Values.auth.jaas.zookeeperUser }} + - name: KAFKA_CFG_SASL_ENABLED_MECHANISMS + value: {{ include "kafka.auth.saslMechanisms" ( dict "type" .Values.auth.saslMechanisms ) }} + - name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL + value: {{ upper .Values.auth.saslInterBrokerMechanism | quote }} + {{- end }} + - name: KAFKA_CFG_LISTENERS + {{- if .Values.listeners }} + value: {{ .Values.listeners }} + {{- else if .Values.externalAccess.enabled }} + value: "INTERNAL://:{{ $interBrokerPort }},CLIENT://:9092,EXTERNAL://:9094" + {{- else }} + value: "INTERNAL://:{{ $interBrokerPort }},CLIENT://:9092" + {{- end }} + {{- if .Values.externalAccess.enabled }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + - name: SHARED_FILE + value: "/shared/info.txt" + {{- end }} + {{- else }} + - name: KAFKA_CFG_ADVERTISED_LISTENERS + {{- if .Values.advertisedListeners }} + value: {{ .Values.advertisedListeners }} + {{- else }} + value: "INTERNAL://$(MY_POD_NAME).{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $interBrokerPort }},CLIENT://$(MY_POD_NAME).{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $clientPort }}" + {{- end }} + {{- end }} + - name: ALLOW_PLAINTEXT_LISTENER + value: {{ ternary "yes" "no" (or .Values.auth.enabled .Values.allowPlaintextListener) | quote }} + {{- if or (include "kafka.client.saslAuthentication" .) (include "kafka.interBroker.saslAuthentication" .) }} + - name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf" + {{- if (include "kafka.client.saslAuthentication" .) }} + - name: KAFKA_CLIENT_USERS + value: {{ join "," .Values.auth.jaas.clientUsers | quote }} + - name: KAFKA_CLIENT_PASSWORDS + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: client-passwords + {{- end }} + {{- if .Values.auth.jaas.zookeeperUser }} + - name: KAFKA_ZOOKEEPER_PROTOCOL + value: "SASL" + - name: KAFKA_ZOOKEEPER_USER + value: {{ .Values.auth.jaas.zookeeperUser | quote }} + - name: KAFKA_ZOOKEEPER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: zookeeper-password + {{- end }} + {{- if (include "kafka.interBroker.saslAuthentication" .) }} + - name: KAFKA_INTER_BROKER_USER + value: {{ .Values.auth.jaas.interBrokerUser | quote }} + - name: KAFKA_INTER_BROKER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: inter-broker-password + {{- end }} + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + - name: KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM + value: {{ .Values.auth.tlsEndpointIdentificationAlgorithm | quote }} + {{- if .Values.auth.jksPassword }} + - name: KAFKA_CERTIFICATE_PASSWORD + value: {{ .Values.auth.jksPassword | quote }} + {{- end }} + {{- end }} + {{- if .Values.metrics.jmx.enabled }} + - name: JMX_PORT + value: "5555" + {{- end }} + - name: KAFKA_CFG_DELETE_TOPIC_ENABLE + value: {{ .Values.deleteTopicEnable | quote }} + - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE + value: {{ .Values.autoCreateTopicsEnable | quote }} + - name: KAFKA_HEAP_OPTS + value: {{ .Values.heapOpts | quote }} + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES + value: {{ .Values.logFlushIntervalMessages | quote }} + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS + value: {{ .Values.logFlushIntervalMs | quote }} + - name: KAFKA_CFG_LOG_RETENTION_BYTES + value: {{ .Values.logRetentionBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS + value: {{ .Values.logRetentionCheckIntervalMs | quote }} + - name: KAFKA_CFG_LOG_RETENTION_HOURS + value: {{ .Values.logRetentionHours | quote }} + - name: KAFKA_CFG_MESSAGE_MAX_BYTES + value: {{ .Values.maxMessageBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_LOG_SEGMENT_BYTES + value: {{ .Values.logSegmentBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_LOG_DIRS + value: {{ .Values.logsDirs | quote }} + - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR + value: {{ .Values.defaultReplicationFactor | quote }} + - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR + value: {{ .Values.offsetsTopicReplicationFactor | quote }} + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: {{ .Values.transactionStateLogReplicationFactor | quote }} + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR + value: {{ .Values.transactionStateLogMinIsr | quote }} + - name: KAFKA_CFG_NUM_IO_THREADS + value: {{ .Values.numIoThreads | quote }} + - name: KAFKA_CFG_NUM_NETWORK_THREADS + value: {{ .Values.numNetworkThreads | quote }} + - name: KAFKA_CFG_NUM_PARTITIONS + value: {{ .Values.numPartitions | quote }} + - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR + value: {{ .Values.numRecoveryThreadsPerDataDir | quote }} + - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES + value: {{ .Values.socketReceiveBufferBytes | quote }} + - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES + value: {{ .Values.socketRequestMaxBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES + value: {{ .Values.socketSendBufferBytes | quote }} + - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS + value: {{ .Values.zookeeperConnectionTimeoutMs | quote }} + {{- if .Values.extraEnvVars }} + {{ include "kafka.tplValue" ( dict "value" .Values.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + ports: + - name: kafka-client + containerPort: 9092 + - name: kafka-internal + containerPort: {{ $interBrokerPort }} + {{- if .Values.externalAccess.enabled }} + - name: kafka-external + containerPort: 9094 + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + {{- else if .Values.customLivenessProbe }} + livenessProbe: {{- include "kafka.tplValue" (dict "value" .Values.customlivenessProbe "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + {{- else if .Values.customReadinessProbe }} + readinessProbe: {{- include "kafka.tplValue" (dict "value" .Values.customreadinessProbe "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/kafka + {{- if or .Values.config .Values.existingConfigmap }} + - name: kafka-config + mountPath: /bitnami/kafka/config/server.properties + subPath: server.properties + {{- end }} + {{- if or .Values.log4j .Values.existingLog4jConfigMap }} + - name: log4j-config + mountPath: /bitnami/kafka/config/log4j.properties + subPath: log4j.properties + {{- end }} + - name: scripts + mountPath: /scripts/setup.sh + subPath: setup.sh + {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }} + - name: shared + mountPath: /shared + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + - name: kafka-certificates + mountPath: /certs + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.metrics.jmx.enabled }} + - name: jmx-exporter + image: {{ template "kafka.metrics.jmx.image" . }} + imagePullPolicy: {{ .Values.metrics.jmx.image.pullPolicy | quote }} + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-kafka/jmx-kafka-prometheus.yml + ports: + - name: metrics + containerPort: 5556 + {{- if .Values.metrics.jmx.resources }} + resources: {{- toYaml .Values.metrics.jmx.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-kafka + {{- end }} + {{- if .Values.sidecars }} + {{- include "kafka.tplValue" (dict "value" .Values.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + {{- if or .Values.config .Values.existingConfigmap }} + - name: kafka-config + configMap: + name: {{ include "kafka.configmapName" . }} + {{- end }} + {{- if or .Values.log4j .Values.existingLog4jConfigMap }} + - name: log4j-config + configMap: + name: {{ include "kafka.log4j.configMapName" . }} + {{ end }} + - name: scripts + configMap: + name: {{ include "kafka.fullname" . }}-scripts + defaultMode: 0755 + {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }} + - name: shared + emptyDir: {} + {{- end }} + {{- if .Values.metrics.jmx.enabled }} + - name: jmx-config + configMap: + name: {{ include "kafka.metrics.jmx.configmapName" . }} + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + - name: kafka-certificates + secret: + secretName: {{ include "kafka.jksSecretName" . }} + defaultMode: 256 + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- if not .Values.persistence.enabled }} + - name: data + emptyDir: {} +{{- else if .Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.persistence.existingClaim .) }} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + {{- if .Values.persistence.annotations }} + annotations: {{- include "kafka.tplValue" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{ include "kafka.storageClass" . | nindent 8 }} +{{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml new file mode 100755 index 000000000..eefe0046d --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml @@ -0,0 +1,52 @@ +{{- if .Values.externalAccess.enabled }} +{{- $fullName := include "kafka.fullname" . }} +{{- $replicaCount := .Values.replicaCount | int }} +{{- $root := . }} + +{{- range $i, $e := until $replicaCount }} +{{- $targetPod := printf "%s-%d" (printf "%s" $fullName) $i }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" $ }}-{{ $i }}-external + labels: {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: kafka + pod: {{ $targetPod }} + {{- if $root.Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" $root.Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or $root.Values.externalAccess.service.annotations $root.Values.commonAnnotations }} + annotations: + {{- if $root.Values.externalAccess.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" $root.Values.externalAccess.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if $root.Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" $root.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ $root.Values.externalAccess.service.type }} + {{- if eq $root.Values.externalAccess.service.type "LoadBalancer" }} + {{- if not (empty $root.Values.externalAccess.service.loadBalancerIPs) }} + loadBalancerIP: {{ index $root.Values.externalAccess.service.loadBalancerIPs $i }} + {{- end }} + {{- if $root.Values.externalAccess.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml $root.Values.externalAccess.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: tcp-kafka + port: {{ $root.Values.externalAccess.service.port }} + {{- if not (empty $root.Values.externalAccess.service.nodePorts) }} + nodePort: {{ index $root.Values.externalAccess.service.nodePorts $i }} + {{- else }} + nodePort: null + {{- end }} + targetPort: kafka-external + selector: {{- include "kafka.matchLabels" $ | nindent 4 }} + app.kubernetes.io/component: kafka + statefulset.kubernetes.io/pod-name: {{ $targetPod }} +--- +{{- end }} +{{- end }} diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml new file mode 100755 index 000000000..e7c2e5e6e --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }}-headless + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-client + port: {{ .Values.service.port }} + protocol: TCP + targetPort: kafka-client + - name: tcp-internal + port: {{ .Values.service.internalPort }} + protocol: TCP + targetPort: kafka-internal + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: kafka diff --git a/ee/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml b/ee/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml new file mode 100755 index 000000000..189cb9ffd --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if eq .Values.service.type "LoadBalancer" }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: tcp-client + port: {{ .Values.service.port }} + protocol: TCP + targetPort: kafka-client + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.client)) }} + nodePort: {{ .Values.service.nodePorts.client }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if and .Values.externalAccess.enabled (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) }} + - name: tcp-external + port: {{ .Values.service.externalPort }} + protocol: TCP + targetPort: kafka-external + {{- if (not (empty .Values.service.nodePorts.external)) }} + nodePort: {{ .Values.service.nodePorts.external }} + {{- end }} + {{- end }} + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: kafka diff --git a/ee/scripts/helm/helm/databases/charts/kafka/values-production.yaml b/ee/scripts/helm/helm/databases/charts/kafka/values-production.yaml new file mode 100755 index 000000000..af6f43dba --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/values-production.yaml @@ -0,0 +1,931 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Kafka image version +## ref: https://hub.docker.com/r/bitnami/kafka/tags/ +## +image: + registry: docker.io + repository: bitnami/kafka + tag: 2.6.0-debian-10-r30 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Set to true if you would like to see extra information on logs + ## + debug: false + +## String to partially override kafka.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override kafka.fullname template +## +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Kafka Configuration +## Specify content for server.properties +## The server.properties is auto-generated based on other parameters when this paremeter is not specified +## +## Example: +## config: |- +## broker.id=-1 +## listeners=PLAINTEXT://:9092 +## advertised.listeners=PLAINTEXT://KAFKA_IP:9092 +## num.network.threads=3 +## num.io.threads=8 +## socket.send.buffer.bytes=102400 +## socket.receive.buffer.bytes=102400 +## socket.request.max.bytes=104857600 +## log.dirs=/bitnami/kafka/data +## num.partitions=1 +## num.recovery.threads.per.data.dir=1 +## offsets.topic.replication.factor=1 +## transaction.state.log.replication.factor=1 +## transaction.state.log.min.isr=1 +## log.flush.interval.messages=10000 +## log.flush.interval.ms=1000 +## log.retention.hours=168 +## log.retention.bytes=1073741824 +## log.segment.bytes=1073741824 +## log.retention.check.interval.ms=300000 +## zookeeper.connect=ZOOKEEPER_SERVICE_NAME +## zookeeper.connection.timeout.ms=6000 +## group.initial.rebalance.delay.ms=0 +## +# config: + +## ConfigMap with Kafka Configuration +## NOTE: This will override config +## +# existingConfigmap: + +## Kafka Log4J Configuration +## An optional log4j.properties file to overwrite the default of the Kafka brokers. +## See an example log4j.properties at: +## https://github.com/apache/kafka/blob/trunk/config/log4j.properties +## +# log4j: + +## Kafka Log4j ConfigMap +## The name of an existing ConfigMap containing a log4j.properties file. +## NOTE: this will override log4j. +## +# existingLog4jConfigMap: + +## Kafka's Java Heap size +## +heapOpts: -Xmx1024m -Xms1024m + +## Switch to enable topic deletion or not. +## +deleteTopicEnable: false + +## Switch to enable auto creation of topics. +## Enabling auto creation of topics not recommended for production or similar environments. +## +autoCreateTopicsEnable: false + +## The number of messages to accept before forcing a flush of data to disk. +## +logFlushIntervalMessages: 10000 + +## The maximum amount of time a message can sit in a log before we force a flush. +## +logFlushIntervalMs: 1000 + +## A size-based retention policy for logs. +## +logRetentionBytes: _1073741824 + +## The interval at which log segments are checked to see if they can be deleted. +## +logRetentionCheckIntervalMs: 300000 + +## The minimum age of a log file to be eligible for deletion due to age. +## +logRetentionHours: 168 + +## The maximum size of a log segment file. When this size is reached a new log segment will be created. +## +logSegmentBytes: _1073741824 + +## A comma separated list of directories under which to store log files. +## +logsDirs: /bitnami/kafka/data + +## The largest record batch size allowed by Kafka +## +maxMessageBytes: _1000012 + +## Default replication factors for automatically created topics +## +defaultReplicationFactor: 3 + +## The replication factor for the offsets topic +## +offsetsTopicReplicationFactor: 3 + +## The replication factor for the transaction topic +## +transactionStateLogReplicationFactor: 3 + +## Overridden min.insync.replicas config for the transaction topic +## +transactionStateLogMinIsr: 3 + +## The number of threads doing disk I/O. +## +numIoThreads: 8 + +## The number of threads handling network requests. +## +numNetworkThreads: 3 + +## The default number of log partitions per topic. +## +numPartitions: 1 + +## The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +## +numRecoveryThreadsPerDataDir: 1 + +## The receive buffer (SO_RCVBUF) used by the socket server. +## +socketReceiveBufferBytes: 102400 + +## The maximum size of a request that the socket server will accept (protection against OOM). +## +socketRequestMaxBytes: _104857600 + +## The send buffer (SO_SNDBUF) used by the socket server. +## +socketSendBufferBytes: 102400 + +## Timeout in ms for connecting to zookeeper. +## +zookeeperConnectionTimeoutMs: 6000 + +## Command and args for running the container. Use array form +## +command: + - /scripts/setup.sh +args: + +## All the parameters from the configuration file can be overwritten by using environment variables with this format: KAFKA_CFG_{KEY} +## ref: https://github.com/bitnami/bitnami-docker-kafka#configuration +## Example: +## extraEnvVars: +## - name: KAFKA_CFG_BACKGROUND_THREADS +## value: "10" +## +extraEnvVars: [] + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Examples: +# extraVolumes: +# - name: kafka-jaas +# secret: +# secretName: kafka-jaas +# extraVolumeMounts: +# - name: kafka-jaas +# mountPath: /bitnami/kafka/config/kafka_jaas.conf +# subPath: kafka_jaas.conf +extraVolumes: [] +extraVolumeMounts: [] + +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: [] + +## Authentication parameteres +## https://github.com/bitnami/bitnami-docker-kafka#security +## +auth: + ## Authentication protocol for client and inter-broker communications + ## Supported values: 'plaintext', 'tls', 'mtls', 'sasl' and 'sasl_tls' + ## This table shows the security provided on each protocol: + ## | Method | Authentication | Encryption via TLS | + ## | plaintext | None | No | + ## | tls | None | Yes | + ## | mtls | Yes (two-way authentication) | Yes | + ## | sasl | Yes (via SASL) | No | + ## | sasl_tls | Yes (via SASL) | Yes | + ## + clientProtocol: sasl + interBrokerProtocol: sasl + + ## Allowed SASL mechanisms when clientProtocol or interBrokerProtocol are using either sasl or sasl_tls + ## + saslMechanisms: plain,scram-sha-256,scram-sha-512 + ## SASL mechanism for inter broker communication + ## + saslInterBrokerMechanism: plain + + ## Name of the existing secret containing the truststore and + ## one keystore per Kafka broker you have in the Kafka cluster. + ## MANDATORY when 'tls', 'mtls', or 'sasl_tls' authentication protocols are used. + ## Create this secret following the steps below: + ## 1) Generate your trustore and keystore files. Helpful script: https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh + ## 2) Rename your truststore to `kafka.truststore.jks`. + ## 3) Rename your keystores to `kafka-X.keystore.jks` where X is the ID of each Kafka broker. + ## 4) Run the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-file=./kafka.truststore.jks --from-file=./kafka-0.keystore.jks --from-file=./kafka-1.keystore.jks ... + ## Alternatively, you can put your JKS files under the files/jks directory + ## + # jksSecret: + + ## Password to access the JKS files when they are password-protected. + ## + # jksPassword: + + ## The endpoint identification algorithm used by clients to validate server host name. + ## Disable server host name verification by setting it to an empty string + ## See: https://docs.confluent.io/current/kafka/authentication_ssl.html#optional-settings + ## + tlsEndpointIdentificationAlgorithm: https + + ## JAAS configuration for SASL authentication + ## MANDATORY when method is 'sasl', or 'sasl_tls' + ## + jaas: + ## Kafka client user list + ## + ## clientUsers: + ## - user1 + ## - user2 + ## + clientUsers: + - user + + ## Kafka client passwords + ## + ## clientPasswords: + ## - password1 + ## - password2 + ## + clientPasswords: [] + + ## Kafka inter broker communication user + ## + interBrokerUser: admin + + ## Kafka inter broker communication password + ## + interBrokerPassword: "" + + ## Kafka Zookeeper user + ## + zookeeperUser: zookeeperUser + + ## Kafka Zookeeper password + ## + zookeeperPassword: zookeeperPassword + + ## Name of the existing secret containing credentials for clientUsers, interBrokerUser and zookeeperUser. + ## Create this secret running the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-literal=client-password=CLIENT_PASSWORD1,CLIENT_PASSWORD2 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=zookeeper-password=ZOOKEEPER_PASSWORD + ## + # existingSecret: + +## The address(es) the socket server listens on. +## When it's set to an empty array, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +listeners: [] + +## The address(es) (hostname:port) the brokers will advertise to producers and consumers. +## When it's set to an empty array, the advertised listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +advertisedListeners: [] + +## The listener->protocol mapping +## When it's nil, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +# listenerSecurityProtocolMap: + +## Allow to use the PLAINTEXT listener. +## +allowPlaintextListener: false + +## Name of listener used for communication between brokers. +## +interBrokerListenerName: INTERNAL + +## Number of Kafka brokers to deploy +## +replicaCount: 3 + +## StrategyType, can be set to RollingUpdate or OnDelete by default. +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## Pod labels. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +## Pod annotations. Evaluated as a template +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} + +## Name of the priority class to be used by kafka pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: "" + +## Affinity for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Kafka pods' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## +podSecurityContext: + fsGroup: 1001 + runAsUser: 1001 + +## Kafka containers' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## Example: +## containerSecurityContext: +## capabilities: +## drop: ["NET_RAW"] +## readOnlyRootFilesystem: true +## +containerSecurityContext: {} + +## Kafka containers' resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 250m + # memory: 1Gi + requests: {} + # cpu: 250m + # memory: 256Mi + +## Kafka containers' liveness and readiness probes. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes +## +livenessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 10 + timeoutSeconds: 5 + # failureThreshold: 3 + # periodSeconds: 10 + # successThreshold: 1 +readinessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 5 + failureThreshold: 6 + timeoutSeconds: 5 + # periodSeconds: 10 + # successThreshold: 1 + +## Pod Disruption Budget configuration +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +pdb: + create: true + ## Min number of pods that must still be available after the eviction + ## + # minAvailable: 1 + ## Max number of pods that can be unavailable after the eviction + ## + maxUnavailable: 1 + +## Add sidecars to the pod. +## Example: +## sidecars: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +sidecars: {} + +## Service parameters +## +service: + ## Service type + ## + type: ClusterIP + ## Kafka port for client connections + ## + port: 9092 + ## Kafka port for inter-broker connections + ## + internalPort: 9093 + ## Kafka port for external connections + ## + externalPort: 9094 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + client: "" + external: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## External Access to Kafka brokers configuration +## +externalAccess: + ## Enable Kubernetes external cluster access to Kafka brokers + ## + enabled: false + + ## External IPs auto-discovery configuration + ## An init container is used to auto-detect LB IPs or node ports by querying the K8s API + ## Note: RBAC might be required + ## + autoDiscovery: + ## Enable external IP/ports auto-discovery + ## + enabled: false + ## Bitnami Kubectl image + ## ref: https://hub.docker.com/r/bitnami/kubectl/tags/ + ## + image: + registry: docker.io + repository: bitnami/kubectl + tag: 1.17.12-debian-10-r3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Parameters to configure K8s service(s) used to externally access Kafka brokers + ## A new service per broker will be created + ## + service: + ## Service type. Allowed values: LoadBalancer or NodePort + ## + type: LoadBalancer + ## Port used when service type is LoadBalancer + ## + port: 9094 + ## Array of load balancer IPs for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## loadBalancerIPs: + ## - X.X.X.X + ## - Y.Y.Y.Y + ## + loadBalancerIPs: [] + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Array of node ports used for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## nodePorts: + ## - 30001 + ## - 30002 + ## + nodePorts: [] + ## When service type is NodePort, you can specify the domain used for Kafka advertised listeners. + ## If not specified, the container will try to get the kubernetes node external IP + ## + # domain: mydomain.com + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## Persistence paramaters +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + ## PV Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. + ## + # storageClass: "-" + ## PV Access Mode + ## + accessModes: + - ReadWriteOnce + ## PVC size + ## + size: 8Gi + ## PVC annotations + ## + annotations: {} + +## Init Container paramaters +## Change the owner and group of the persistent volume(s) mountpoint(s) to 'runAsUser:fsGroup' on each component +## values from the securityContext section of the component +## +volumePermissions: + enabled: false + ## Bitnami Minideb image + ## ref: https://hub.docker.com/r/bitnami/minideb/tags/ + ## + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## Kafka pods ServiceAccount +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: true + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fluentd.fullname template + ## + # name: + +## Role Based Access +## ref: https://kubernetes.io/docs/admin/authorization/rbac/ +## +rbac: + ## Specifies whether RBAC rules should be created + ## binding Kafka ServiceAccount to a role + ## that allows Kafka pods querying the K8s API + ## + create: false + +## Prometheus Exporters / Metrics +## +metrics: + ## Prometheus Kafka Exporter: exposes complimentary metrics to JMX Exporter + ## + kafka: + enabled: true + + ## Bitnami Kafka exporter image + ## ref: https://hub.docker.com/r/bitnami/kafka-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/kafka-exporter + tag: 1.2.0-debian-10-r220 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Extra flags to be passed to Kafka exporter + ## Example: + ## extraFlags: + ## tls.insecure-skip-tls-verify: "" + ## web.telemetry-path: "/metrics" + ## + extraFlags: {} + + ## Name of the existing secret containing the optional certificate and key files + ## for Kafka Exporter client authentication + ## + # certificatesSecret: + + ## Prometheus Kafka Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## Kafka Exporter Service type + ## + type: ClusterIP + ## Kafka Exporter Prometheus port + ## + port: 9308 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the Kafka Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.kafka.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus JMX Exporter: exposes the majority of Kafkas metrics + ## + jmx: + enabled: true + + ## Bitnami JMX exporter image + ## ref: https://hub.docker.com/r/bitnami/jmx-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/jmx-exporter + tag: 0.14.0-debian-10-r15 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Prometheus JMX Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## JMX Exporter Service type + ## + type: ClusterIP + ## JMX Exporter Prometheus port + ## + port: 5556 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the JMX Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.jmx.service.port }}" + prometheus.io/path: "/" + + ## JMX Whitelist Objects, can be set to control which JMX metrics are exposed. Only whitelisted + ## values will be exposed via JMX Exporter. They must also be exposed via Rules. To expose all metrics + ## (warning its crazy excessive and they aren't formatted in a prometheus style) (1) `whitelistObjectNames: []` + ## (2) commented out above `overrideConfig`. + ## + whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* + + ## Prometheus JMX exporter configuration + ## Specify content for jmx-kafka-prometheus.yml. Evaluated as a template + ## + ## Credits to the incubator/kafka chart for the JMX configuration. + ## https://github.com/helm/charts/tree/master/incubator/kafka + ## + config: |- + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false + {{- if .Values.metrics.jmx.whitelistObjectNames }} + whitelistObjectNames: ["{{ join "\",\"" .Values.metrics.jmx.whitelistObjectNames }}"] + {{- end }} + + ## ConfigMap with Prometheus JMX exporter configuration + ## NOTE: This will override metrics.jmx.config + ## + # existingConfigmap: + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace in which Prometheus is running + ## + # namespace: monitoring + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + +## +## Zookeeper chart configuration +## +## https://github.com/bitnami/charts/blob/master/bitnami/zookeeper/values.yaml +## +zookeeper: + enabled: true + auth: + ## Enable Zookeeper auth + ## + enabled: true + ## User that will use Zookeeper clients to auth + ## + clientUser: zookeeperUser + ## Password that will use Zookeeper clients to auth + ## + clientPassword: zookeeperPassword + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + serverUsers: zookeeperUser + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + serverPasswords: zookeeperPassword + metrics: + enabled: true + +## This value is only used when zookeeper.enabled is set to false +## +externalZookeeper: + ## Server or list of external zookeeper servers to use. + ## + servers: [] diff --git a/ee/scripts/helm/helm/databases/charts/kafka/values.yaml b/ee/scripts/helm/helm/databases/charts/kafka/values.yaml new file mode 100755 index 000000000..154d71bd5 --- /dev/null +++ b/ee/scripts/helm/helm/databases/charts/kafka/values.yaml @@ -0,0 +1,934 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Kafka image version +## ref: https://hub.docker.com/r/bitnami/kafka/tags/ +## +image: + registry: docker.io + repository: bitnami/kafka + tag: 2.6.0-debian-10-r30 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Set to true if you would like to see extra information on logs + ## + debug: false + +## String to partially override kafka.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override kafka.fullname template +## +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Kafka Configuration +## Specify content for server.properties +## The server.properties is auto-generated based on other parameters when this paremeter is not specified +## +## Example: +## config: |- +## broker.id=-1 +## listeners=PLAINTEXT://:9092 +## advertised.listeners=PLAINTEXT://KAFKA_IP:9092 +## num.network.threads=3 +## num.io.threads=8 +## socket.send.buffer.bytes=102400 +## socket.receive.buffer.bytes=102400 +## socket.request.max.bytes=104857600 +## log.dirs=/bitnami/kafka/data +## num.partitions=1 +## num.recovery.threads.per.data.dir=1 +## offsets.topic.replication.factor=1 +## transaction.state.log.replication.factor=1 +## transaction.state.log.min.isr=1 +## log.flush.interval.messages=10000 +## log.flush.interval.ms=1000 +## log.retention.hours=168 +## log.retention.bytes=1073741824 +## log.segment.bytes=1073741824 +## log.retention.check.interval.ms=300000 +## zookeeper.connect=ZOOKEEPER_SERVICE_NAME +## zookeeper.connection.timeout.ms=6000 +## group.initial.rebalance.delay.ms=0 +## +# config: + +## ConfigMap with Kafka Configuration +## NOTE: This will override config +## +# existingConfigmap: + +## Kafka Log4J Configuration +## An optional log4j.properties file to overwrite the default of the Kafka brokers. +## See an example log4j.properties at: +## https://github.com/apache/kafka/blob/trunk/config/log4j.properties +## +# log4j: + +## Kafka Log4j ConfigMap +## The name of an existing ConfigMap containing a log4j.properties file. +## NOTE: this will override log4j. +## +# existingLog4jConfigMap: + +## Kafka's Java Heap size +## +heapOpts: -Xmx1024m -Xms1024m + +## Switch to enable topic deletion or not. +## +deleteTopicEnable: false + +## Switch to enable auto creation of topics. +## Enabling auto creation of topics not recommended for production or similar environments. +## +autoCreateTopicsEnable: true + +## The number of messages to accept before forcing a flush of data to disk. +## +logFlushIntervalMessages: 10000 + +## The maximum amount of time a message can sit in a log before we force a flush. +## +logFlushIntervalMs: 1000 + +## A size-based retention policy for logs. +## +logRetentionBytes: _1073741824 + +## The interval at which log segments are checked to see if they can be deleted. +## +logRetentionCheckIntervalMs: 300000 + +## The minimum age of a log file to be eligible for deletion due to age. +## +logRetentionHours: 168 + +## The maximum size of a log segment file. When this size is reached a new log segment will be created. +## +logSegmentBytes: _1073741824 + +## A comma separated list of directories under which to store log files. +## +logsDirs: /bitnami/kafka/data + +## The largest record batch size allowed by Kafka +## +maxMessageBytes: _1000012 + +## Default replication factors for automatically created topics +## +defaultReplicationFactor: 1 + +## The replication factor for the offsets topic +## +offsetsTopicReplicationFactor: 1 + +## The replication factor for the transaction topic +## +transactionStateLogReplicationFactor: 1 + +## Overridden min.insync.replicas config for the transaction topic +## +transactionStateLogMinIsr: 1 + +## The number of threads doing disk I/O. +## +numIoThreads: 8 + +## The number of threads handling network requests. +## +numNetworkThreads: 3 + +## The default number of log partitions per topic. +## +numPartitions: 1 + +## The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +## +numRecoveryThreadsPerDataDir: 1 + +## The receive buffer (SO_RCVBUF) used by the socket server. +## +socketReceiveBufferBytes: 102400 + +## The maximum size of a request that the socket server will accept (protection against OOM). +## +socketRequestMaxBytes: _104857600 + +## The send buffer (SO_SNDBUF) used by the socket server. +## +socketSendBufferBytes: 102400 + +## Timeout in ms for connecting to zookeeper. +## +zookeeperConnectionTimeoutMs: 6000 + +## Command and args for running the container. Use array form +## +command: + - /scripts/setup.sh +args: + +## All the parameters from the configuration file can be overwritten by using environment variables with this format: KAFKA_CFG_{KEY} +## ref: https://github.com/bitnami/bitnami-docker-kafka#configuration +## Example: +## extraEnvVars: +## - name: KAFKA_CFG_BACKGROUND_THREADS +## value: "10" +## +extraEnvVars: [] + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Examples: +# extraVolumes: +# - name: kafka-jaas +# secret: +# secretName: kafka-jaas +# extraVolumeMounts: +# - name: kafka-jaas +# mountPath: /bitnami/kafka/config/kafka_jaas.conf +# subPath: kafka_jaas.conf +extraVolumes: [] +extraVolumeMounts: [] + +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: [] + +## Authentication parameteres +## https://github.com/bitnami/bitnami-docker-kafka#security +## +auth: + ## Authentication protocol for client and inter-broker communications + ## Supported values: 'plaintext', 'tls', 'mtls', 'sasl' and 'sasl_tls' + ## This table shows the security provided on each protocol: + ## | Method | Authentication | Encryption via TLS | + ## | plaintext | None | No | + ## | tls | None | Yes | + ## | mtls | Yes (two-way authentication) | Yes | + ## | sasl | Yes (via SASL) | No | + ## | sasl_tls | Yes (via SASL) | Yes | + ## + clientProtocol: plaintext + interBrokerProtocol: plaintext + + ## Allowed SASL mechanisms when clientProtocol or interBrokerProtocol are using either sasl or sasl_tls + ## + saslMechanisms: plain,scram-sha-256,scram-sha-512 + ## SASL mechanism for inter broker communication + ## + saslInterBrokerMechanism: plain + + ## Name of the existing secret containing the truststore and + ## one keystore per Kafka broker you have in the Kafka cluster. + ## MANDATORY when 'tls', 'mtls', or 'sasl_tls' authentication protocols are used. + ## Create this secret following the steps below: + ## 1) Generate your trustore and keystore files. Helpful script: https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh + ## 2) Rename your truststore to `kafka.truststore.jks`. + ## 3) Rename your keystores to `kafka-X.keystore.jks` where X is the ID of each Kafka broker. + ## 4) Run the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-file=./kafka.truststore.jks --from-file=./kafka-0.keystore.jks --from-file=./kafka-1.keystore.jks ... + ## Alternatively, you can put your JKS files under the files/jks directory + ## + # jksSecret: + + ## Password to access the JKS files when they are password-protected. + ## + # jksPassword: + + ## The endpoint identification algorithm used by clients to validate server host name. + ## Disable server host name verification by setting it to an empty string + ## See: https://docs.confluent.io/current/kafka/authentication_ssl.html#optional-settings + ## + tlsEndpointIdentificationAlgorithm: https + + ## JAAS configuration for SASL authentication + ## MANDATORY when method is 'sasl', or 'sasl_tls' + ## + jaas: + ## Kafka client user list + ## + ## clientUsers: + ## - user1 + ## - user2 + ## + clientUsers: + - user + + ## Kafka client passwords. This is mandatory if more than one user is specified in clientUsers. + ## + ## clientPasswords: + ## - password1 + ## - password2" + ## + clientPasswords: [] + + ## Kafka inter broker communication user + ## + interBrokerUser: admin + + ## Kafka inter broker communication password + ## + interBrokerPassword: "" + + ## Kafka Zookeeper user + ## + # zookeeperUser: + + ## Kafka Zookeeper password + ## + # zookeeperPassword: + + ## Name of the existing secret containing credentials for clientUsers, interBrokerUser and zookeeperUser. + ## Create this secret running the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1,CLIENT_PASSWORD2 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=zookeeper-password=ZOOKEEPER_PASSWORD + ## + # existingSecret: + +## The address(es) the socket server listens on. +## When it's set to an empty array, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +listeners: [] + +## The address(es) (hostname:port) the brokers will advertise to producers and consumers. +## When it's set to an empty array, the advertised listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +advertisedListeners: [] + +## The listener->protocol mapping +## When it's nil, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +# listenerSecurityProtocolMap: + +## Allow to use the PLAINTEXT listener. +## +allowPlaintextListener: true + +## Name of listener used for communication between brokers. +## +interBrokerListenerName: INTERNAL + +## Number of Kafka brokers to deploy +## +replicaCount: 2 + +## StrategyType, can be set to RollingUpdate or OnDelete by default. +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## Pod labels. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +## +podLabels: {} + +## Pod annotations. Evaluated as a template +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} + +## Name of the priority class to be used by kafka pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +priorityClassName: "" + +## Affinity for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Kafka pods' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## +podSecurityContext: + fsGroup: 1001 + runAsUser: 1001 + +## Kafka containers' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## Example: +## containerSecurityContext: +## capabilities: +## drop: ["NET_RAW"] +## readOnlyRootFilesystem: true +## +containerSecurityContext: {} + +## Kafka containers' resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 250m + # memory: 1Gi + requests: {} + # cpu: 250m + # memory: 256Mi + +## Kafka containers' liveness and readiness probes. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes +## +livenessProbe: + enabled: true + initialDelaySeconds: 10 + timeoutSeconds: 5 + # failureThreshold: 3 + # periodSeconds: 10 + # successThreshold: 1 +readinessProbe: + enabled: true + initialDelaySeconds: 5 + failureThreshold: 6 + timeoutSeconds: 5 + # periodSeconds: 10 + # successThreshold: 1 + +## Custom liveness/readiness probes that will override the default ones +## +customLivenessProbe: {} +customReadinessProbe: {} + +## Pod Disruption Budget configuration +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +pdb: + create: true + ## Min number of pods that must still be available after the eviction + ## + # minAvailable: 1 + ## Max number of pods that can be unavailable after the eviction + ## + maxUnavailable: 1 + +## Add sidecars to the pod. +## Example: +## sidecars: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +sidecars: {} + +## Service parameters +## +service: + ## Service type + ## + type: ClusterIP + ## Kafka port for client connections + ## + port: 9092 + ## Kafka port for inter-broker connections + ## + internalPort: 9093 + ## Kafka port for external connections + ## + externalPort: 9094 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + client: "" + external: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## External Access to Kafka brokers configuration +## +externalAccess: + ## Enable Kubernetes external cluster access to Kafka brokers + ## + enabled: false + + ## External IPs auto-discovery configuration + ## An init container is used to auto-detect LB IPs or node ports by querying the K8s API + ## Note: RBAC might be required + ## + autoDiscovery: + ## Enable external IP/ports auto-discovery + ## + enabled: false + ## Bitnami Kubectl image + ## ref: https://hub.docker.com/r/bitnami/kubectl/tags/ + ## + image: + registry: docker.io + repository: bitnami/kubectl + tag: 1.17.12-debian-10-r3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Parameters to configure K8s service(s) used to externally access Kafka brokers + ## A new service per broker will be created + ## + service: + ## Service type. Allowed values: LoadBalancer or NodePort + ## + type: LoadBalancer + ## Port used when service type is LoadBalancer + ## + port: 9094 + ## Array of load balancer IPs for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## loadBalancerIPs: + ## - X.X.X.X + ## - Y.Y.Y.Y + ## + loadBalancerIPs: [] + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Array of node ports used for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## nodePorts: + ## - 30001 + ## - 30002 + ## + nodePorts: [] + ## When service type is NodePort, you can specify the domain used for Kafka advertised listeners. + ## If not specified, the container will try to get the kubernetes node external IP + ## + # domain: mydomain.com + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## Persistence paramaters +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + ## PV Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. + ## + # storageClass: "-" + ## PV Access Mode + ## + accessModes: + - ReadWriteOnce + ## PVC size + ## + size: 8Gi + ## PVC annotations + ## + annotations: {} + +## Init Container paramaters +## Change the owner and group of the persistent volume(s) mountpoint(s) to 'runAsUser:fsGroup' on each component +## values from the securityContext section of the component +## +volumePermissions: + enabled: false + ## Bitnami Minideb image + ## ref: https://hub.docker.com/r/bitnami/minideb/tags/ + ## + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## Kafka pods ServiceAccount +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: true + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fluentd.fullname template + ## + # name: + +## Role Based Access +## ref: https://kubernetes.io/docs/admin/authorization/rbac/ +## +rbac: + ## Specifies whether RBAC rules should be created + ## binding Kafka ServiceAccount to a role + ## that allows Kafka pods querying the K8s API + ## + create: false + +## Prometheus Exporters / Metrics +## +metrics: + ## Prometheus Kafka Exporter: exposes complimentary metrics to JMX Exporter + ## + kafka: + enabled: false + + ## Bitnami Kafka exporter image + ## ref: https://hub.docker.com/r/bitnami/kafka-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/kafka-exporter + tag: 1.2.0-debian-10-r220 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Extra flags to be passed to Kafka exporter + ## Example: + ## extraFlags: + ## tls.insecure-skip-tls-verify: "" + ## web.telemetry-path: "/metrics" + ## + extraFlags: {} + + ## Name of the existing secret containing the optional certificate and key files + ## for Kafka Exporter client authentication + ## + # certificatesSecret: + + ## Prometheus Kafka Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## Kafka Exporter Service type + ## + type: ClusterIP + ## Kafka Exporter Prometheus port + ## + port: 9308 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the Kafka Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.kafka.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus JMX Exporter: exposes the majority of Kafkas metrics + ## + jmx: + enabled: false + + ## Bitnami JMX exporter image + ## ref: https://hub.docker.com/r/bitnami/jmx-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/jmx-exporter + tag: 0.14.0-debian-10-r15 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Prometheus JMX Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## JMX Exporter Service type + ## + type: ClusterIP + ## JMX Exporter Prometheus port + ## + port: 5556 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the JMX Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.jmx.service.port }}" + prometheus.io/path: "/" + + ## JMX Whitelist Objects, can be set to control which JMX metrics are exposed. Only whitelisted + ## values will be exposed via JMX Exporter. They must also be exposed via Rules. To expose all metrics + ## (warning its crazy excessive and they aren't formatted in a prometheus style) (1) `whitelistObjectNames: []` + ## (2) commented out above `overrideConfig`. + ## + whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* + + ## Prometheus JMX exporter configuration + ## Specify content for jmx-kafka-prometheus.yml. Evaluated as a template + ## + ## Credits to the incubator/kafka chart for the JMX configuration. + ## https://github.com/helm/charts/tree/master/incubator/kafka + ## + config: |- + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false + {{- if .Values.metrics.jmx.whitelistObjectNames }} + whitelistObjectNames: ["{{ join "\",\"" .Values.metrics.jmx.whitelistObjectNames }}"] + {{- end }} + + ## ConfigMap with Prometheus JMX exporter configuration + ## NOTE: This will override metrics.jmx.config + ## + # existingConfigmap: + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace in which Prometheus is running + ## + # namespace: monitoring + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + +## +## Zookeeper chart configuration +## +## https://github.com/bitnami/charts/blob/master/bitnami/zookeeper/values.yaml +## +zookeeper: + enabled: true + auth: + ## Enable Zookeeper auth + ## + enabled: false + ## User that will use Zookeeper clients to auth + ## + # clientUser: + ## Password that will use Zookeeper clients to auth + ## + # clientPassword: + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + # serverUsers: + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + # serverPasswords: + +## This value is only used when zookeeper.enabled is set to false +## +externalZookeeper: + ## Server or list of external zookeeper servers to use. + ## + servers: [] diff --git a/scripts/helm/helm/README.md b/scripts/helm/helm/README.md new file mode 100644 index 000000000..ee63b1567 --- /dev/null +++ b/scripts/helm/helm/README.md @@ -0,0 +1,14 @@ +- Initialize databases + - we've to pass the --wait flag, else the db installation won't be complete. and it'll break the db init. + - collate all dbs required + - How to distinguish b/w enterprise and community + - Or fist only community then enterprise +- install db migration + - have to have another helm chart with low hook value for higher prioriry +- install app + - customize values.yaml file + + +## Installation +helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic +helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./values.yaml --atomic diff --git a/scripts/helm/helm/databases/.helmignore b/scripts/helm/helm/databases/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/databases/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/databases/Chart.yaml b/scripts/helm/helm/databases/Chart.yaml new file mode 100644 index 000000000..0aa62594a --- /dev/null +++ b/scripts/helm/helm/databases/Chart.yaml @@ -0,0 +1,42 @@ +apiVersion: v2 +name: databases +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" + +dependencies: + - name: kafka + repository: file://charts/kafka + version: 11.8.6 + condition: kafka.enabled + - name: clickhouse + repository: file://charts/clickhouse + version: 1.16.0 + condition: clickhouse.enabled + - name: postgresql + repository: file://charts/postgresql + version: 9.8.2 + condition: postgresql.enabled + - name: redis + repository: file://charts/redis + version: 12.10.1 + condition: redis.enabled diff --git a/scripts/helm/helm/databases/charts/clickhouse/.helmignore b/scripts/helm/helm/databases/charts/clickhouse/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml b/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml new file mode 100644 index 000000000..c7a0eb3d6 --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: clickhouse +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.16.0 diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl b/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl new file mode 100644 index 000000000..44cfadff0 --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "clickhouse.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "clickhouse.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "clickhouse.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "clickhouse.labels" -}} +helm.sh/chart: {{ include "clickhouse.chart" . }} +{{ include "clickhouse.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "clickhouse.selectorLabels" -}} +app.kubernetes.io/name: {{ include "clickhouse.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "clickhouse.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "clickhouse.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml b/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml new file mode 100644 index 000000000..4496f556c --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + labels: + {{- include "clickhouse.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.webPort }} + targetPort: web + protocol: TCP + name: web + - port: {{ .Values.service.dataPort }} + targetPort: data + protocol: TCP + name: data + selector: + {{- include "clickhouse.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml b/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml new file mode 100644 index 000000000..1f1183598 --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "clickhouse.serviceAccountName" . }} + labels: + {{- include "clickhouse.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml b/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml new file mode 100644 index 000000000..392976eec --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "clickhouse.fullname" . }} + labels: + {{- include "clickhouse.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + serviceName: {{ include "clickhouse.fullname" . }} + selector: + matchLabels: + {{- include "clickhouse.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "clickhouse.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "clickhouse.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + env: + {{- range $key, $value := .Values.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 9000 + name: web + - containerPort: 8123 + name: data + volumeMounts: + - name: ch-volume + mountPath: /var/lib/mydata + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: ch-volume + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.storageSize }} diff --git a/scripts/helm/helm/databases/charts/clickhouse/values.yaml b/scripts/helm/helm/databases/charts/clickhouse/values.yaml new file mode 100644 index 000000000..4cba1c1f8 --- /dev/null +++ b/scripts/helm/helm/databases/charts/clickhouse/values.yaml @@ -0,0 +1,62 @@ +# Default values for clickhouse. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: yandex/clickhouse-server + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "20.9" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +env: {} + +service: + webPort: 9000 + dataPort: 8123 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} +storageSize: 8G diff --git a/scripts/helm/helm/databases/charts/kafka/.helmignore b/scripts/helm/helm/databases/charts/kafka/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/scripts/helm/helm/databases/charts/kafka/Chart.yaml b/scripts/helm/helm/databases/charts/kafka/Chart.yaml new file mode 100755 index 000000000..165e70d55 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + category: Infrastructure +apiVersion: v1 +appVersion: 2.6.0 +description: Apache Kafka is a distributed streaming platform. +engine: gotpl +home: https://github.com/bitnami/charts/tree/master/bitnami/kafka +icon: https://bitnami.com/assets/stacks/kafka/img/kafka-stack-110x117.png +keywords: +- kafka +- zookeeper +- streaming +- producer +- consumer +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: kafka +sources: +- https://github.com/bitnami/bitnami-docker-kafka +- https://kafka.apache.org/ +version: 11.8.6 diff --git a/scripts/helm/helm/databases/charts/kafka/README.md b/scripts/helm/helm/databases/charts/kafka/README.md new file mode 100755 index 000000000..5584bd43d --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/README.md @@ -0,0 +1,737 @@ +# Kafka + +[Kafka](https://www.kafka.org/) is a distributed streaming platform used for building real-time data pipelines and streaming apps. It is horizontally scalable, fault-tolerant, wicked fast, and runs in production in thousands of companies. + +## TL;DR + +```console +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install my-release bitnami/kafka +``` + +## Introduction + +This chart bootstraps a [Kafka](https://github.com/bitnami/bitnami-docker-kafka) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install my-release bitnami/kafka +``` + +These commands deploy Kafka on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +The following tables lists the configurable parameters of the Kafka chart and their default values per section/component: + +### Global parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | + +### Common parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `nameOverride` | String to partially override kafka.fullname | `nil` | +| `fullnameOverride` | String to fully override kafka.fullname | `nil` | +| `clusterDomain` | Default Kubernetes cluster domain | `cluster.local` | +| `commonLabels` | Labels to add to all deployed objects | `{}` | +| `commonAnnotations` | Annotations to add to all deployed objects | `{}` | +| `extraDeploy` | Array of extra objects to deploy with the release | `nil` (evaluated as a template) | + +### Kafka parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `image.registry` | Kafka image registry | `docker.io` | +| `image.repository` | Kafka image name | `bitnami/kafka` | +| `image.tag` | Kafka image tag | `{TAG_NAME}` | +| `image.pullPolicy` | Kafka image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Set to true if you would like to see extra information on logs | `false` | +| `config` | Configuration file for Kafka. Auto-generated based on other parameters when not specified | `nil` | +| `existingConfigmap` | Name of existing ConfigMap with Kafka configuration | `nil` | +| `log4j` | An optional log4j.properties file to overwrite the default of the Kafka brokers. | `nil` | +| `existingLog4jConfigMap` | The name of an existing ConfigMap containing a log4j.properties file. | `nil` | +| `heapOpts` | Kafka's Java Heap size | `-Xmx1024m -Xms1024m` | +| `deleteTopicEnable` | Switch to enable topic deletion or not | `false` | +| `autoCreateTopicsEnable` | Switch to enable auto creation of topics. Enabling auto creation of topics not recommended for production or similar environments | `false` | +| `logFlushIntervalMessages` | The number of messages to accept before forcing a flush of data to disk | `10000` | +| `logFlushIntervalMs` | The maximum amount of time a message can sit in a log before we force a flush | `1000` | +| `logRetentionBytes` | A size-based retention policy for logs | `_1073741824` | +| `logRetentionCheckIntervalMs` | The interval at which log segments are checked to see if they can be deleted | `300000` | +| `logRetentionHours` | The minimum age of a log file to be eligible for deletion due to age | `168` | +| `logSegmentBytes` | The maximum size of a log segment file. When this size is reached a new log segment will be created | `_1073741824` | +| `logsDirs` | A comma separated list of directories under which to store log files | `/bitnami/kafka/data` | +| `maxMessageBytes` | The largest record batch size allowed by Kafka | `1000012` | +| `defaultReplicationFactor` | Default replication factors for automatically created topics | `1` | +| `offsetsTopicReplicationFactor` | The replication factor for the offsets topic | `1` | +| `transactionStateLogReplicationFactor` | The replication factor for the transaction topic | `1` | +| `transactionStateLogMinIsr` | Overridden min.insync.replicas config for the transaction topic | `1` | +| `numIoThreads` | The number of threads doing disk I/O | `8` | +| `numNetworkThreads` | The number of threads handling network requests | `3` | +| `numPartitions` | The default number of log partitions per topic | `1` | +| `numRecoveryThreadsPerDataDir` | The number of threads per data directory to be used for log recovery at startup and flushing at shutdown | `1` | +| `socketReceiveBufferBytes` | The receive buffer (SO_RCVBUF) used by the socket server | `102400` | +| `socketRequestMaxBytes` | The maximum size of a request that the socket server will accept (protection against OOM) | `_104857600` | +| `socketSendBufferBytes` | The send buffer (SO_SNDBUF) used by the socket server | `102400` | +| `zookeeperConnectionTimeoutMs` | Timeout in ms for connecting to Zookeeper | `6000` | +| `extraEnvVars` | Extra environment variables to add to kafka pods | `[]` | +| `extraVolumes` | Extra volume(s) to add to Kafka statefulset | `[]` | +| `extraVolumeMounts` | Extra volumeMount(s) to add to Kafka containers | `[]` | +| `auth.clientProtocol` | Authentication protocol for communications with clients. Allowed protocols: `plaintext`, `tls`, `mtls`, `sasl` and `sasl_tls` | `plaintext` | +| `auth.interBrokerProtocol` | Authentication protocol for inter-broker communications. Allowed protocols: `plaintext`, `tls`, `mtls`, `sasl` and `sasl_tls` | `plaintext` | +| `auth.saslMechanisms` | SASL mechanisms when either `auth.interBrokerProtocol` or `auth.clientProtocol` are `sasl`. Allowed types: `plain`, `scram-sha-256`, `scram-sha-512` | `plain,scram-sha-256,scram-sha-512` | +| `auth.saslInterBrokerMechanism` | SASL mechanism to use as inter broker protocol, it must be included at `auth.saslMechanisms` | `plain` | +| `auth.jksSecret` | Name of the existing secret containing the truststore and one keystore per Kafka broker you have in the cluster | `nil` | +| `auth.jksPassword` | Password to access the JKS files when they are password-protected | `nil` | +| `auth.tlsEndpointIdentificationAlgorithm` | The endpoint identification algorithm to validate server hostname using server certificate | `https` | +| `auth.jaas.interBrokerUser` | Kafka inter broker communication user for SASL authentication | `admin` | +| `auth.jaas.interBrokerPassword` | Kafka inter broker communication password for SASL authentication | `nil` | +| `auth.jaas.zookeeperUser` | Kafka Zookeeper user for SASL authentication | `nil` | +| `auth.jaas.zookeeperPassword` | Kafka Zookeeper password for SASL authentication | `nil` | +| `auth.jaas.existingSecret` | Name of the existing secret containing credentials for brokerUser, interBrokerUser and zookeeperUser | `nil` | +| `auth.jaas.clientUsers` | List of Kafka client users to be created, separated by commas. This values will override `auth.jaas.clientUser` | `[]` | +| `auth.jaas.clientPasswords` | List of passwords for `auth.jaas.clientUsers`. It is mandatory to provide the passwords when using `auth.jaas.clientUsers` | `[]` | +| `listeners` | The address(es) the socket server listens on. Auto-calculated it's set to an empty array | `[]` | +| `advertisedListeners` | The address(es) (hostname:port) the broker will advertise to producers and consumers. Auto-calculated it's set to an empty array | `[]` | +| `listenerSecurityProtocolMap` | The protocol->listener mapping. Auto-calculated it's set to nil | `nil` | +| `allowPlaintextListener` | Allow to use the PLAINTEXT listener | `true` | +| `interBrokerListenerName` | The listener that the brokers should communicate on | `INTERNAL` | + +### Statefulset parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | Number of Kafka nodes | `1` | +| `updateStrategy` | Update strategy for the stateful set | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `podLabels` | Kafka pod labels | `{}` (evaluated as a template) | +| `podAnnotations` | Kafka Pod annotations | `{}` (evaluated as a template) | +| `affinity` | Affinity for pod assignment | `{}` (evaluated as a template) | +| `priorityClassName` | Name of the existing priority class to be used by kafka pods | `""` | +| `nodeSelector` | Node labels for pod assignment | `{}` (evaluated as a template) | +| `tolerations` | Tolerations for pod assignment | `[]` (evaluated as a template) | +| `podSecurityContext` | Kafka pods' Security Context | `{}` | +| `containerSecurityContext` | Kafka containers' Security Context | `{}` | +| `resources.limits` | The resources limits for Kafka containers | `{}` | +| `resources.requests` | The requested resources for Kafka containers | `{}` | +| `livenessProbe` | Liveness probe configuration for Kafka | `Check values.yaml file` | +| `readinessProbe` | Readiness probe configuration for Kafka | `Check values.yaml file` | +| `customLivenessProbe` | Custom Liveness probe configuration for Kafka | `{}` | +| `customReadinessProbe` | Custom Readiness probe configuration for Kafka | `{}` | +| `pdb.create` | Enable/disable a Pod Disruption Budget creation | `false` | +| `pdb.minAvailable` | Minimum number/percentage of pods that should remain scheduled | `nil` | +| `pdb.maxUnavailable` | Maximum number/percentage of pods that may be made unavailable | `1` | +| `command` | Override kafka container command | `['/scripts/setup.sh']` (evaluated as a template) | +| `args` | Override kafka container arguments | `[]` (evaluated as a template) | +| `sidecars` | Attach additional sidecar containers to the Kafka pod | `{}` | + +### Exposure parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | Kafka port for client connections | `9092` | +| `service.internalPort` | Kafka port for inter-broker connections | `9093` | +| `service.externalPort` | Kafka port for external connections | `9094` | +| `service.nodePorts.client` | Nodeport for client connections | `""` | +| `service.nodePorts.external` | Nodeport for external connections | `""` | +| `service.loadBalancerIP` | loadBalancerIP for Kafka Service | `nil` | +| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]` | +| `service.annotations` | Service annotations | `{}`(evaluated as a template) | +| `externalAccess.enabled` | Enable Kubernetes external cluster access to Kafka brokers | `false` | +| `externalAccess.autoDiscovery.enabled` | Enable using an init container to auto-detect external IPs/ports by querying the K8s API | `false` | +| `externalAccess.autoDiscovery.image.registry` | Init container auto-discovery image registry (kubectl) | `docker.io` | +| `externalAccess.autoDiscovery.image.repository` | Init container auto-discovery image name (kubectl) | `bitnami/kubectl` | +| `externalAccess.autoDiscovery.image.tag` | Init container auto-discovery image tag (kubectl) | `{TAG_NAME}` | +| `externalAccess.autoDiscovery.image.pullPolicy` | Init container auto-discovery image pull policy (kubectl) | `Always` | +| `externalAccess.autoDiscovery.resources.limits` | Init container auto-discovery resource limits | `{}` | +| `externalAccess.autoDiscovery.resources.requests` | Init container auto-discovery resource requests | `{}` | +| `externalAccess.service.type` | Kubernetes Service type for external access. It can be NodePort or LoadBalancer | `LoadBalancer` | +| `externalAccess.service.port` | Kafka port used for external access when service type is LoadBalancer | `9094` | +| `externalAccess.service.loadBalancerIPs` | Array of load balancer IPs for Kafka brokers | `[]` | +| `externalAccess.service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]` | +| `externalAccess.service.domain` | Domain or external ip used to configure Kafka external listener when service type is NodePort | `nil` | +| `externalAccess.service.nodePorts` | Array of node ports used to configure Kafka external listener when service type is NodePort | `[]` | +| `externalAccess.service.annotations` | Service annotations for external access | `{}`(evaluated as a template) | + +### Persistence parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `persistence.enabled` | Enable Kafka data persistence using PVC, note that Zookeeper persistence is unaffected | `true` | +| `persistence.existingClaim` | Provide an existing `PersistentVolumeClaim`, the value is evaluated as a template | `nil` | +| `persistence.storageClass` | PVC Storage Class for Kafka data volume | `nil` | +| `persistence.accessMode` | PVC Access Mode for Kafka data volume | `ReadWriteOnce` | +| `persistence.size` | PVC Storage Request for Kafka data volume | `8Gi` | +| `persistence.annotations` | Annotations for the PVC | `{}`(evaluated as a template) | + +### RBAC parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `serviceAccount.create` | Enable creation of ServiceAccount for Kafka pods | `true` | +| `serviceAccount.name` | Name of the created serviceAccount | Generated using the `kafka.fullname` template | +| `rbac.create` | Weather to create & use RBAC resources or not | `false` | + +### Volume Permissions parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `volumePermissions.enabled` | Enable init container that changes the owner and group of the persistent volume(s) mountpoint to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/minideb` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `buster` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `volumePermissions.resources.limits` | Init container volume-permissions resource limits | `{}` | +| `volumePermissions.resources.requests` | Init container volume-permissions resource requests | `{}` | + +### Metrics parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `metrics.kafka.enabled` | Whether or not to create a standalone Kafka exporter to expose Kafka metrics | `false` | +| `metrics.kafka.image.registry` | Kafka exporter image registry | `docker.io` | +| `metrics.kafka.image.repository` | Kafka exporter image name | `bitnami/kafka-exporter` | +| `metrics.kafka.image.tag` | Kafka exporter image tag | `{TAG_NAME}` | +| `metrics.kafka.image.pullPolicy` | Kafka exporter image pull policy | `IfNotPresent` | +| `metrics.kafka.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.kafka.extraFlags` | Extra flags to be passed to Kafka exporter | `{}` | +| `metrics.kafka.certificatesSecret` | Name of the existing secret containing the optional certificate and key files | `nil` | +| `metrics.kafka.resources.limits` | Kafka Exporter container resource limits | `{}` | +| `metrics.kafka.resources.requests` | Kafka Exporter container resource requests | `{}` | +| `metrics.kafka.service.type` | Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for Kafka Exporter | `ClusterIP` | +| `metrics.kafka.service.port` | Kafka Exporter Prometheus port | `9308` | +| `metrics.kafka.service.nodePort` | Kubernetes HTTP node port | `""` | +| `metrics.kafka.service.annotations` | Annotations for Prometheus metrics service | `Check values.yaml file` | +| `metrics.kafka.service.loadBalancerIP` | loadBalancerIP if service type is `LoadBalancer` | `nil` | +| `metrics.kafka.service.clusterIP` | Static clusterIP or None for headless services | `nil` | +| `metrics.jmx.enabled` | Whether or not to expose JMX metrics to Prometheus | `false` | +| `metrics.jmx.image.registry` | JMX exporter image registry | `docker.io` | +| `metrics.jmx.image.repository` | JMX exporter image name | `bitnami/jmx-exporter` | +| `metrics.jmx.image.tag` | JMX exporter image tag | `{TAG_NAME}` | +| `metrics.jmx.image.pullPolicy` | JMX exporter image pull policy | `IfNotPresent` | +| `metrics.jmx.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `metrics.jmx.resources.limits` | JMX Exporter container resource limits | `{}` | +| `metrics.jmx.resources.requests` | JMX Exporter container resource requests | `{}` | +| `metrics.jmx.service.type` | Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for JMX Exporter | `ClusterIP` | +| `metrics.jmx.service.port` | JMX Exporter Prometheus port | `5556` | +| `metrics.jmx.service.nodePort` | Kubernetes HTTP node port | `""` | +| `metrics.jmx.service.annotations` | Annotations for Prometheus metrics service | `Check values.yaml file` | +| `metrics.jmx.service.loadBalancerIP` | loadBalancerIP if service type is `LoadBalancer` | `nil` | +| `metrics.jmx.service.clusterIP` | Static clusterIP or None for headless services | `nil` | +| `metrics.jmx.whitelistObjectNames` | Allows setting which JMX objects you want to expose to via JMX stats to JMX Exporter | (see `values.yaml`) | +| `metrics.jmx.config` | Configuration file for JMX exporter | (see `values.yaml`) | +| `metrics.jmx.existingConfigmap` | Name of existing ConfigMap with JMX exporter configuration | `nil` | +| `metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.kafka.enabled` or `metrics.jmx.enabled` to be `true`) | `false` | +| `metrics.serviceMonitor.namespace` | Namespace which Prometheus is running in | `monitoring` | +| `metrics.serviceMonitor.interval` | Interval at which metrics should be scraped | `nil` | +| `metrics.serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `nil` (Prometheus Operator default value) | +| `metrics.serviceMonitor.selector` | ServiceMonitor selector labels | `nil` (Prometheus Operator default value) | + +### Zookeeper chart parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `zookeeper.enabled` | Switch to enable or disable the Zookeeper helm chart | `true` | +| `zookeeper.persistence.enabled` | Enable Zookeeper persistence using PVC | `true` | +| `externalZookeeper.servers` | Server or list of external Zookeeper servers to use | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +helm install my-release \ + --set replicaCount=3 \ + bitnami/kafka +``` + +The above command deploys Kafka with 3 brokers (replicas). + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +helm install my-release -f values.yaml bitnami/kafka +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Production configuration and horizontal scaling + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one. + +- Number of Kafka nodes: + +```diff +- replicaCount: 1 ++ replicaCount: 3 +``` + +- Allow to use the PLAINTEXT listener: + +```diff +- allowPlaintextListener: true ++ allowPlaintextListener: false +``` + +- Default replication factors for automatically created topics: + +```diff +- defaultReplicationFactor: 1 ++ defaultReplicationFactor: 3 +``` + +- Allow auto creation of topics. + +```diff +- autoCreateTopicsEnable: true ++ autoCreateTopicsEnable: false +``` + +- The replication factor for the offsets topic: + +```diff +- offsetsTopicReplicationFactor: 1 ++ offsetsTopicReplicationFactor: 3 +``` + +- The replication factor for the transaction topic: + +```diff +- transactionStateLogReplicationFactor: 1 ++ transactionStateLogReplicationFactor: 3 +``` + +- Overridden min.insync.replicas config for the transaction topic: + +```diff +- transactionStateLogMinIsr: 1 ++ transactionStateLogMinIsr: 3 +``` + +- Switch to enable the Kafka SASAL authentication on client and inter-broker communications: + +```diff +- auth.clientProtocol: plaintext ++ auth.clientProtocol: sasl +- auth.interBrokerProtocol: plaintext ++ auth.interBrokerProtocol: sasl +``` + +- Enable Zookeeper authentication: + +```diff ++ auth.jaas.zookeeperUser: zookeeperUser ++ auth.jaas.zookeeperPassword: zookeeperPassword +- zookeeper.auth.enabled: false ++ zookeeper.auth.enabled: true ++ zookeeper.auth.clientUser: zookeeperUser ++ zookeeper.auth.clientPassword: zookeeperPassword ++ zookeeper.auth.serverUsers: zookeeperUser ++ zookeeper.auth.serverPasswords: zookeeperPassword +``` + +- Enable Pod Disruption Budget: + +```diff +- pdb.create: false ++ pdb.create: true +``` + +- Create a separate Kafka metrics exporter: + +```diff +- metrics.kafka.enabled: false ++ metrics.kafka.enabled: true +``` + +- Expose JMX metrics to Prometheus: + +```diff +- metrics.jmx.enabled: false ++ metrics.jmx.enabled: true +``` + +- Enable Zookeeper metrics: + +```diff ++ zookeeper.metrics.enabled: true +``` + +To horizontally scale this chart once it has been deployed, you can upgrade the statefulset using a new value for the `replicaCount` parameter. Please note that, when enabling TLS encryption, you must update your JKS secret including the keystore for the new replicas. + +### Setting custom parameters + +Any environment variable beginning with `KAFKA_CFG_` will be mapped to its corresponding Kafka key. For example, use `KAFKA_CFG_BACKGROUND_THREADS` in order to set `background.threads`. In order to pass custom environment variables use the `extraEnvVars` property. + +### Listeners configuration + +This chart allows you to automatically configure Kafka with 3 listeners: + +- One for inter-broker communications. +- A second one for communications with clients within the K8s cluster. +- (optional) a third listener for communications with clients outside the K8s cluster. Check [this section](#accessing-kafka-brokers-from-outside-the-clusters) for more information. + +For more complex configurations, set the `listeners`, `advertisedListeners` and `listenerSecurityProtocolMap` parameters as needed. + +### Enable security for Kafka and Zookeeper + +You can configure different authentication protocols for each listener you configure in Kafka. For instance, you can use `sasl_tls` authentication for client communications, while using `tls` for inter-broker communications. This table shows the available protocols and the security they provide: + +| Method | Authentication | Encryption via TLS | +|-----------|-------------------------------|--------------------| +| plaintext | None | No | +| tls | None | Yes | +| mtls | Yes (two-way authentication) | Yes | +| sasl | Yes (via SASL) | No | +| sasl_tls | Yes (via SASL) | Yes | + +If you enabled SASL authentication on any listener, you can set the SASL credentials using the parameters below: + +- `auth.jaas.clientUsers`/`auth.jaas.clientPasswords`: when enabling SASL authentication for communications with clients. +- `auth.jaas.interBrokerUser`/`auth.jaas.interBrokerPassword`: when enabling SASL authentication for inter-broker communications. +- `auth.jaas.zookeeperUser`/`auth.jaas.zookeeperPassword`: In the case that the Zookeeper chart is deployed with SASL authentication enabled. + +In order to configure TLS authentication/encryption, you **must** create a secret containing the Java Key Stores (JKS) files: the truststore (`kafka.truststore.jks`) and one keystore (`kafka.keystore.jks`) per Kafka broker you have in the cluster. Then, you need pass the secret name with the `--auth.jksSecret` parameter when deploying the chart. + +> **Note**: If the JKS files are password protected (recommended), you will need to provide the password to get access to the keystores. To do so, use the `auth.jksPassword` parameter to provide your password. + +For instance, to configure TLS authentication on a Kafka cluster with 2 Kafka brokers use the command below to create the secret: + +```console +kubectl create secret generic kafka-jks --from-file=./kafka.truststore.jks --from-file=./kafka-0.keystore.jks --from-file=./kafka-1.keystore.jks +``` + +> **Note**: the command above assumes you already created the trustore and keystores files. This [script](https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh) can help you with the JKS files generation. + +As an alternative to manually create the secret before installing the chart, you can put your JKS files inside the chart folder `files/jks`, an a secret including them will be generated. Please note this alternative requires to have the chart downloaded locally, so you will have to clone this repository or fetch the chart before installing it. + +You can deploy the chart with authentication using the following parameters: + +```console +replicaCount=2 +auth.clientProtocol=sasl +auth.interBrokerProtocol=tls +auth.certificatesSecret=kafka-jks +auth.certificatesPassword=jksPassword +auth.jaas.clientUsers[0]=brokerUser +auth.jaas.clientPassword[0]=brokerPassword +auth.jaas.zookeeperUser=zookeeperUser +auth.jaas.zookeeperPassword=zookeeperPassword +zookeeper.auth.enabled=true +zookeeper.auth.serverUsers=zookeeperUser +zookeeper.auth.serverPasswords=zookeeperPassword +zookeeper.auth.clientUser=zookeeperUser +zookeeper.auth.clientPassword=zookeeperPassword +``` + +If you also enable exposing metrics using the Kafka expoter, and you are using `sasl_tls`, `tls`, or `mtls` authentication protocols, you need to mount the CA certificated used to sign the brokers certificates in the exporter so it can validate the Kafka brokers. To do so, create a secret containing the CA, and set the `metrics.certificatesSecret` parameter. As an alternative, you can skip TLS validation using extra flags: + +```console +metrics.kafka.extraFlags={tls.insecure-skip-tls-verify: ""} +``` + +### Accessing Kafka brokers from outside the cluster + +In order to access Kafka Brokers from outside the cluster, an additional listener and advertised listener must be configured. Additionally, a specific service per kafka pod will be created. + +There are two ways of configuring external access. Using LoadBalancer services or using NodePort services. + +#### Using LoadBalancer services + +You have two alternatives to use LoadBalancer services: + +- Option A) Use random load balancer IPs using an **initContainer** that waits for the IPs to be ready and discover them automatically. + +```console +externalAccess.enabled=true +externalAccess.service.type=LoadBalancer +externalAccess.service.port=9094 +externalAccess.autoDiscovery.enabled=true +serviceAccount.create=true +rbac.create=true +``` + +Note: This option requires creating RBAC rules on clusters where RBAC policies are enabled. + +- Option B) Manually specify the load balancer IPs: + +```console +externalAccess.enabled=true +externalAccess.service.type=LoadBalancer +externalAccess.service.port=9094 +externalAccess.service.loadBalancerIPs[0]='external-ip-1' +externalAccess.service.loadBalancerIPs[1]='external-ip-2'} +``` + +Note: You need to know in advance the load balancer IPs so each Kafka broker advertised listener is configured with it. + +#### Using NodePort services + +You have two alternatives to use NodePort services: + +- Option A) Use random node ports using an **initContainer** that discover them automatically. + +```console +externalAccess.enabled=true +externalAccess.service.type=NodePort +externalAccess.autoDiscovery.enabled=true +serviceAccount.create=true +rbac.create=true +``` + +Note: This option requires creating RBAC rules on clusters where RBAC policies are enabled. + +- Option B) Manually specify the node ports: + +```console +externalAccess.enabled=true +externalAccess.service.type=NodePort +externalAccess.serivce.nodePorts[0]='node-port-1' +externalAccess.serivce.nodePorts[1]='node-port-2' +``` + +Note: You need to know in advance the node ports that will be exposed so each Kafka broker advertised listener is configured with it. + +The pod will try to get the external ip of the node using `curl -s https://ipinfo.io/ip` unless `externalAccess.service.domain` is provided. + +Following the aforementioned steps will also allow to connect the brokers from the outside using the cluster's default service (when `service.type` is `LoadBalancer` or `NodePort`). Use the property `service.externalPort` to specify the port used for external connections. + +### Sidecars + +If you have a need for additional containers to run within the same pod as Kafka (e.g. an additional metrics or logging exporter), you can do so via the `sidecars` config parameter. Simply define your container according to the Kubernetes container spec. + +```yaml +sidecars: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +``` + +### Deploying extra resources + +There are cases where you may want to deploy extra objects, such as Kafka Connect. For covering this case, the chart allows adding the full specification of other objects using the `extraDeploy` parameter. The following example would create a deployment including a Kafka Connect deployment so you can connect Kafka with MongoDB: + +```yaml +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: |- + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: {{ include "kafka.fullname" . }}-connect + labels: {{- include "kafka.labels" . | nindent 6 }} + app.kubernetes.io/component: connector + spec: + replicas: 1 + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 8 }} + app.kubernetes.io/component: connector + template: + metadata: + labels: {{- include "kafka.labels" . | nindent 10 }} + app.kubernetes.io/component: connector + spec: + containers: + - name: connect + image: KAFKA-CONNECT-IMAGE + imagePullPolicy: IfNotPresent + ports: + - name: connector + containerPort: 8083 + volumeMounts: + - name: configuration + mountPath: /opt/bitnami/kafka/config + volumes: + - name: configuration + configMap: + name: {{ include "kafka.fullname" . }}-connect + - apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ include "kafka.fullname" . }}-connect + labels: {{- include "kafka.labels" . | nindent 6 }} + app.kubernetes.io/component: connector + data: + connect-standalone.properties: |- + bootstrap.servers = {{ include "kafka.fullname" . }}-0.{{ include "kafka.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}:{{ .Values.service.port }} + ... + mongodb.properties: |- + connection.uri=mongodb://root:password@mongodb-hostname:27017 + ... + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kafka.fullname" . }}-connect + labels: {{- include "kafka.labels" . | nindent 6 }} + app.kubernetes.io/component: connector + spec: + ports: + - protocol: TCP + port: 8083 + targetPort: connector + selector: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: connector +``` + +You can create the Kafka Connect image using the Dockerfile below: + +```Dockerfile +FROM bitnami/kafka:latest +# Download MongoDB Connector for Apache Kafka https://www.confluent.io/hub/mongodb/kafka-connect-mongodb +RUN mkdir -p /opt/bitnami/kafka/plugins && \ + cd /opt/bitnami/kafka/plugins && \ + curl --remote-name --location --silent https://search.maven.org/remotecontent?filepath=org/mongodb/kafka/mongo-kafka-connect/1.2.0/mongo-kafka-connect-1.2.0-all.jar +CMD /opt/bitnami/kafka/bin/connect-standalone.sh /opt/bitnami/kafka/config/connect-standalone.properties /opt/bitnami/kafka/config/mongo.properties +``` + +## Persistence + +The [Bitnami Kafka](https://github.com/bitnami/bitnami-docker-kafka) image stores the Kafka data at the `/bitnami/kafka` path of the container. + +Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. See the [Parameters](#persistence-parameters) section to configure the PVC or to disable persistence. + +### Adjust permissions of persistent volume mountpoint + +As the image run as non-root by default, it is necessary to adjust the ownership of the persistent volume so that the container can write data into it. + +By default, the chart is configured to use Kubernetes Security Context to automatically change the ownership of the volume. However, this feature does not work in all Kubernetes distributions. +As an alternative, this chart supports using an initContainer to change the ownership of the volume before mounting it in the final destination. + +You can enable this initContainer by setting `volumePermissions.enabled` to `true`. + +## Upgrading + +### To 11.8.0 + +External access to brokers can now be archived through the cluster's Kafka service. + +- `service.nodePort` -> deprecated in favor of `service.nodePorts.client` and `service.nodePorts.external` + +### To 11.7.0 + +The way to configure the users and passwords changed. Now it is allowed to create multiple users during the installation by providing the list of users and passwords. + +- `auth.jaas.clientUser` (string) -> deprecated in favor of `auth.jaas.clientUsers` (array). +- `auth.jaas.clientPassword` (string) -> deprecated in favor of `auth.jaas.clientPasswords` (array). + +### To 11.0.0 + +The way to configure listeners and authentication on Kafka is totally refactored allowing users to configure different authentication protocols on different listeners. Please check the sections [Listeners Configuration](listeners-configuration) and [Listeners Configuration](enable-kafka-for-kafka-and-zookeeper) for more information. + +Backwards compatibility is not guaranteed you adapt your values.yaml to the new format. Here you can find some parameters that were renamed or disappeared in favor of new ones on this major version: + +- `auth.enabled` -> deprecated in favor of `auth.clientProtocol` and `auth.interBrokerProtocol` parameters. +- `auth.ssl` -> deprecated in favor of `auth.clientProtocol` and `auth.interBrokerProtocol` parameters. +- `auth.certificatesSecret` -> renamed to `auth.jksSecret`. +- `auth.certificatesPassword` -> renamed to `auth.jksPassword`. +- `sslEndpointIdentificationAlgorithm` -> renamedo to `auth.tlsEndpointIdentificationAlgorithm`. +- `auth.interBrokerUser` -> renamed to `auth.jaas.interBrokerUser` +- `auth.interBrokerPassword` -> renamed to `auth.jaas.interBrokerPassword` +- `auth.zookeeperUser` -> renamed to `auth.jaas.zookeeperUser` +- `auth.zookeeperPassword` -> renamed to `auth.jaas.zookeeperPassword` +- `auth.existingSecret` -> renamed to `auth.jaas.existingSecret` +- `service.sslPort` -> deprecated in favor of `service.internalPort` +- `service.nodePorts.kafka` and `service.nodePorts.ssl` -> deprecated in favor of `service.nodePort` +- `metrics.kafka.extraFlag` -> new parameter +- `metrics.kafka.certificatesSecret` -> new parameter + +### To 10.0.0 + +If you are setting the `config` or `log4j` parameter, backwards compatibility is not guaranteed, because the `KAFKA_MOUNTED_CONFDIR` has moved from `/opt/bitnami/kafka/conf` to `/bitnami/kafka/config`. In order to continue using these parameters, you must also upgrade your image to `docker.io/bitnami/kafka:2.4.1-debian-10-r38` or later. + +### To 9.0.0 + +Backwards compatibility is not guaranteed you adapt your values.yaml to the new format. Here you can find some parameters that were renamed on this major version: + +```diff +- securityContext.enabled +- securityContext.fsGroup +- securityContext.fsGroup ++ podSecurityContext +- externalAccess.service.loadBalancerIP ++ externalAccess.service.loadBalancerIPs +- externalAccess.service.nodePort ++ externalAccess.service.nodePorts +- metrics.jmx.configMap.enabled +- metrics.jmx.configMap.overrideConfig ++ metrics.jmx.config +- metrics.jmx.configMap.overrideName ++ metrics.jmx.existingConfigmap +``` + +Ports names were prefixed with the protocol to comply with Istio (see https://istio.io/docs/ops/deployment/requirements/). + +### To 8.0.0 + +There is not backwards compatibility since the brokerID changes to the POD_NAME. For more information see [this PR](https://github.com/bitnami/charts/pull/2028). + +### To 7.0.0 + +Backwards compatibility is not guaranteed when Kafka metrics are enabled, unless you modify the labels used on the exporter deployments. +Use the workaround below to upgrade from versions previous to 7.0.0. The following example assumes that the release name is kafka: + +```console +helm upgrade kafka bitnami/kafka --version 6.1.8 --set metrics.kafka.enabled=false +helm upgrade kafka bitnami/kafka --version 7.0.0 --set metrics.kafka.enabled=true +``` + +### To 2.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's deployments. +Use the workaround below to upgrade from versions previous to 2.0.0. The following example assumes that the release name is kafka: + +```console +kubectl delete statefulset kafka-kafka --cascade=false +kubectl delete statefulset kafka-zookeeper --cascade=false +``` + +### To 1.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's deployments. +Use the workaround below to upgrade from versions previous to 1.0.0. The following example assumes that the release name is kafka: + +```console +kubectl delete statefulset kafka-kafka --cascade=false +kubectl delete statefulset kafka-zookeeper --cascade=false +``` diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml new file mode 100755 index 000000000..c3b15dc5c --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml @@ -0,0 +1,20 @@ +annotations: + category: Infrastructure +apiVersion: v1 +appVersion: 3.6.2 +description: A centralized service for maintaining configuration information, naming, + providing distributed synchronization, and providing group services for distributed + applications. +engine: gotpl +home: https://github.com/bitnami/charts/tree/master/bitnami/zookeeper +icon: https://bitnami.com/assets/stacks/zookeeper/img/zookeeper-stack-110x117.png +keywords: +- zookeeper +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: zookeeper +sources: +- https://github.com/bitnami/bitnami-docker-zookeeper +- https://zookeeper.apache.org/ +version: 5.21.9 diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md new file mode 100755 index 000000000..0291875ed --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md @@ -0,0 +1,297 @@ +# ZooKeeper + +[ZooKeeper](https://zookeeper.apache.org/) is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. All of these kinds of services are used in some form or other by distributed applications. + +## TL;DR + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/zookeeper +``` + +## Introduction + +This chart bootstraps a [ZooKeeper](https://github.com/bitnami/bitnami-docker-zookeeper) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/zookeeper +``` + +These commands deploy ZooKeeper on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +The following tables lists the configurable parameters of the ZooKeeper chart and their default values per section/component: + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | + +### Common parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `nameOverride` | String to partially override zookeeper.fullname | `nil` | +| `fullnameOverride` | String to fully override zookeeper.fullname | `nil` | +| `clusterDomain` | Default Kubernetes cluster domain | `cluster.local` | +| `commonLabels` | Labels to add to all deployed objects | `{}` | +| `commonAnnotations` | Annotations to add to all deployed objects | `{}` | +| `schedulerName` | Kubernetes pod scheduler registry | `nil` (use the default-scheduler) | + +### Zookeeper chart parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `image.registry` | ZooKeeper image registry | `docker.io` | +| `image.repository` | ZooKeeper Image name | `bitnami/zookeeper` | +| `image.tag` | ZooKeeper Image tag | `{TAG_NAME}` | +| `image.pullPolicy` | ZooKeeper image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug values should be set | `false` | +| `tickTime` | Basic time unit in milliseconds used by ZooKeeper for heartbeats | `2000` | +| `initLimit` | Time the ZooKeeper servers in quorum have to connect to a leader | `10` | +| `syncLimit` | How far out of date a server can be from a leader | `5` | +| `maxClientCnxns` | Number of concurrent connections that a single client may make to a single member | `60` | +| `maxSessionTimeout` | Maximum session timeout in milliseconds that the server will allow the client to negotiate. | `40000` | +| `autopurge.snapRetainCount` | Number of retains snapshots for autopurge | `3` | +| `autopurge.purgeInterval` | The time interval in hours for which the purge task has to be triggered | `0` | +| `fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr, mntr` | +| `listenOnAllIPs` | Allow Zookeeper to listen for connections from its peers on all available IP addresses. | `false` | +| `allowAnonymousLogin` | Allow to accept connections from unauthenticated users | `yes` | +| `auth.existingSecret` | Use existing secret (ignores previous password) | `nil` | +| `auth.enabled` | Enable ZooKeeper auth | `false` | +| `auth.clientUser` | User that will use ZooKeeper clients to auth | `nil` | +| `auth.clientPassword` | Password that will use ZooKeeper clients to auth | `nil` | +| `auth.serverUsers` | List of user to be created | `nil` | +| `auth.serverPasswords` | List of passwords to assign to users when created | `nil` | +| `heapSize` | Size in MB for the Java Heap options (Xmx and XMs) | `[]` | +| `logLevel` | Log level of ZooKeeper server | `ERROR` | +| `jvmFlags` | Default JVMFLAGS for the ZooKeeper process | `nil` | +| `config` | Configure ZooKeeper with a custom zoo.conf file | `nil` | +| `dataLogDir` | Data log directory | `""` | + +### Statefulset parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | Number of ZooKeeper nodes | `1` | +| `updateStrategy` | Update strategy for the statefulset | `RollingUpdate` | +| `rollingUpdatePartition` | Partition update strategy | `nil` | +| `podManagementPolicy` | Pod management policy | `Parallel` | +| `podLabels` | ZooKeeper pod labels | `{}` (evaluated as a template) | +| `podAnnotations` | ZooKeeper Pod annotations | `{}` (evaluated as a template) | +| `affinity` | Affinity for pod assignment | `{}` (evaluated as a template) | +| `nodeSelector` | Node labels for pod assignment | `{}` (evaluated as a template) | +| `tolerations` | Tolerations for pod assignment | `[]` (evaluated as a template) | +| `priorityClassName` | Name of the existing priority class to be used by ZooKeeper pods | `""` | +| `securityContext.enabled` | Enable security context (ZooKeeper master pod) | `true` | +| `securityContext.fsGroup` | Group ID for the container (ZooKeeper master pod) | `1001` | +| `securityContext.runAsUser` | User ID for the container (ZooKeeper master pod) | `1001` | +| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `livenessProbe` | Liveness probe configuration for ZooKeeper | Check `values.yaml` file | +| `readinessProbe` | Readiness probe configuration for ZooKeeper | Check `values.yaml` file | +| `extraVolumes` | Extra volumes | `nil` | +| `extraVolumeMounts` | Mount extra volume(s) | `nil` | +| `podDisruptionBudget.maxUnavailable` | Max number of pods down simultaneously | `1` | + +### Exposure parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | ZooKeeper port | `2181` | +| `service.followerPort` | ZooKeeper follower port | `2888` | +| `service.electionPort` | ZooKeeper election port | `3888` | +| `service.publishNotReadyAddresses` | If the ZooKeeper headless service should publish DNS records for not ready pods | `true` | +| `serviceAccount.create` | Enable creation of ServiceAccount for zookeeper pod | `false` | +| `serviceAccount.name` | The name of the service account to use. If not set and `create` is `true`, a name is generated | Generated using the `zookeeper.fullname` template | +| `service.tls.client_enable` | Enable tls for client connections | `false` | +| `service.tls.quorum_enable` | Enable tls for quorum protocol | `false` | +| `service.tls.disable_base_client_port` | Remove client port from service definitions. | `false` | +| `service.tls.client_port` | Service port for tls client connections | `3181` | +| `service.tls.client_keystore_path` | KeyStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_key_store/key_store_file` | +| `service.tls.client_keystore_password` | KeyStore password. You can use environment variables. | `nil` | +| `service.tls.client_truststore_path` | TrustStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_trust_store/trust_store_file` | +| `service.tls.client_truststore_password` | TrustStore password. You can use environment variables. | `nil` | +| `service.tls.quorum_keystore_path` | KeyStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_key_store/key_store_file` | +| `service.tls.quorum_keystore_password` | KeyStore password. You can use environment variables. | `nil` | +| `service.tls.quorum_truststore_path` | TrustStore file path. Refer to extraVolumes and extraVolumeMounts for mounting files into the pods | `/tls_trust_store/trust_store_file` | +| `service.tls.quorum_truststore_password` | TrustStore password. You can use environment variables. | `nil` | +| `service.annotations` | Annotations for the Service | `{}` | +| `service.headless.annotations` | Annotations for the Headless Service | `{}` | +| `networkPolicy.enabled` | Enable NetworkPolicy | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | + +### Persistence parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `persistence.enabled` | Enable Zookeeper data persistence using PVC | `true` | +| `persistence.existingClaim` | Provide an existing `PersistentVolumeClaim` | `nil` (evaluated as a template) | +| `persistence.storageClass` | PVC Storage Class for ZooKeeper data volume | `nil` | +| `persistence.accessMode` | PVC Access Mode for ZooKeeper data volume | `ReadWriteOnce` | +| `persistence.size` | PVC Storage Request for ZooKeeper data volume | `8Gi` | +| `persistence.annotations` | Annotations for the PVC | `{}` (evaluated as a template) | +| `persistence.dataLogDir.size` | PVC Storage Request for ZooKeeper's Data log directory | `8Gi` | +| `persistence.dataLogDir.existingClaim` | Provide an existing `PersistentVolumeClaim` for Zookeeper's Data log directory | `nil` (evaluated as a template) | + +### Volume Permissions parameters + +| Parameter | Description | Default | +|---------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `volumePermissions.enabled` | Enable init container that changes the owner and group of the persistent volume(s) mountpoint to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/minideb` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `buster` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.resources` | Init container resource requests/limit | `nil` | + +### Metrics parameters + +| Parameter | Description | Default | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------| +| `metrics.enabled` | Enable prometheus to access zookeeper metrics endpoint | `false` | +| `metrics.containerPort` | Port where a Jetty server will expose Prometheus metrics | `9141` | +| `metrics.service.type` | Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for Jetty server exposing Prometheus metrics | `ClusterIP` | +| `metrics.service.port` | Prometheus metrics service port | `9141` | +| `metrics.service.annotations` | Service annotations for Prometheus to auto-discover the metrics endpoint | `{prometheus.io/scrape: "true", prometheus.io/port: "9141"}` | +| `metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (also requires `metrics.enabled` to be `true`) | `false` | +| `metrics.serviceMonitor.namespace` | Namespace for the ServiceMonitor Resource | The Release Namespace | +| `metrics.serviceMonitor.interval` | Interval at which metrics should be scraped. | `nil` (Prometheus Operator default value) | +| `metrics.serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `nil` (Prometheus Operator default value) | +| `metrics.serviceMonitor.selector` | Prometheus instance selector labels | `nil` | +| `metrics.prometheusRule.enabled` | if `true`, creates a Prometheus Operator PrometheusRule (also requires `metrics.enabled` to be `true` and `metrics.prometheusRule.rules`) | `false` | +| `metrics.prometheusRule.namespace` | Namespace for the PrometheusRule Resource | The Release Namespace | +| `metrics.prometheusRule.selector` | Prometheus instance selector labels | `nil` | +| `metrics.prometheusRule.rules` | Prometheus Rule definitions (see values.yaml for examples) | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install my-release \ + --set auth.clientUser=newUser \ + bitnami/zookeeper +``` + +The above command sets the ZooKeeper user to `newUser`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +$ helm install my-release -f values.yaml bitnami/zookeeper +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Production configuration + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one. + +- Number of ZooKeeper nodes: + +```diff +- replicaCount: 1 ++ replicaCount: 3 +``` + +- Enable prometheus metrics: + +```diff +- metrics.enabled: false ++ metrics.enabled: true +``` + +### Log level + +You can configure the ZooKeeper log level using the `ZOO_LOG_LEVEL` environment variable. By default, it is set to `ERROR` because of each readiness probe produce an `INFO` message on connection and a `WARN` message on disconnection. + +## Persistence + +The [Bitnami ZooKeeper](https://github.com/bitnami/bitnami-docker-zookeeper) image stores the ZooKeeper data and configurations at the `/bitnami/zookeeper` path of the container. + +Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. +See the [Parameters](#parameters) section to configure the PVC or to disable persistence. + +### Adjust permissions of persistent volume mountpoint + +As the image run as non-root by default, it is necessary to adjust the ownership of the persistent volume so that the container can write data into it. + +By default, the chart is configured to use Kubernetes Security Context to automatically change the ownership of the volume. However, this feature does not work in all Kubernetes distributions. +As an alternative, this chart supports using an initContainer to change the ownership of the volume before mounting it in the final destination. + +You can enable this initContainer by setting `volumePermissions.enabled` to `true`. + +### Data Log Directory + +You can use a dedicated device for logs (instead of using the data directory) to help avoiding competition between logging and snapshots. To do so, set the `dataLogDir` parameter with the path to be used for writing transaction logs. Alternatively, set this parameter with an empty string an it result in the log being written to the data directory (Zookeeper's default behavior). + +When using a dedicated device for logs, you can use a PVC to persist the logs. To do so, set `persistence.enabled` to `true`. See the [Persistence Parameters](#persistence-parameters) section for more information. + +## Upgrading + +### To 5.21.0 + +A couple of parameters related to Zookeeper metrics were renamed or disappeared in favor of new ones: + +- `metrics.port` is renamed to `metrics.containerPort`. +- `metrics.annotations` is deprecated in favor of `metrics.service.annotations`. + +### To 3.0.0 + +This new version of the chart includes the new ZooKeeper major version 3.5.5. Note that to perform an automatic upgrade +of the application, each node will need to have at least one snapshot file created in the data directory. If not, the +new version of the application won't be able to start the service. Please refer to [ZOOKEEPER-3056](https://issues.apache.org/jira/browse/ZOOKEEPER-3056) +in order to find ways to workaround this issue in case you are facing it. + +### To 2.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's statefulsets. +Use the workaround below to upgrade from versions previous to 2.0.0. The following example assumes that the release name is `zookeeper`: + +```console +$ kubectl delete statefulset zookeeper-zookeeper --cascade=false +``` + +### To 1.0.0 + +Backwards compatibility is not guaranteed unless you modify the labels used on the chart's deployments. +Use the workaround below to upgrade from versions previous to 1.0.0. The following example assumes that the release name is zookeeper: + +```console +$ kubectl delete statefulset zookeeper-zookeeper --cascade=false +``` diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt new file mode 100755 index 000000000..3cc2edbed --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt @@ -0,0 +1,57 @@ +{{- if contains .Values.service.type "LoadBalancer" }} +{{- if not .Values.auth.clientPassword }} +------------------------------------------------------------------------------- + WARNING + + By specifying "serviceType=LoadBalancer" and not specifying "auth.enabled=true" + you have most likely exposed the ZooKeeper service externally without any + authentication mechanism. + + For security reasons, we strongly suggest that you switch to "ClusterIP" or + "NodePort". As alternative, you can also specify a valid password on the + "auth.clientPassword" parameter. + +------------------------------------------------------------------------------- +{{- end }} +{{- end }} + +** Please be patient while the chart is being deployed ** + +ZooKeeper can be accessed via port 2181 on the following DNS name from within your cluster: + + {{ template "zookeeper.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + +To connect to your ZooKeeper server run the following commands: + + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "zookeeper.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=zookeeper" -o jsonpath="{.items[0].metadata.name}") + kubectl exec -it $POD_NAME -- zkCli.sh + +To connect to your ZooKeeper server from outside the cluster execute the following commands: + +{{- if contains "NodePort" .Values.service.type }} + + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "zookeeper.fullname" . }}) + zkCli.sh $NODE_IP:$NODE_PORT + +{{- else if contains "LoadBalancer" .Values.service.type }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "zookeeper.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "zookeeper.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + zkCli.sh $SERVICE_IP:2181 + +{{- else if contains "ClusterIP" .Values.service.type }} + + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "zookeeper.fullname" . }} 2181:2181 & + zkCli.sh 127.0.0.1:2181 + +{{- end }} + +{{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }} + +WARNING: Rolling tag detected ({{ .Values.image.repository }}:{{ .Values.image.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ + +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl new file mode 100755 index 000000000..f82502d69 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl @@ -0,0 +1,212 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "zookeeper.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "zookeeper.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "zookeeper.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* + Create the name of the service account to use + */}} +{{- define "zookeeper.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "zookeeper.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Zookeeper image name +*/}} +{{- define "zookeeper.image" -}} +{{- $registryName := .Values.image.registry -}} +{{- $repositoryName := .Values.image.repository -}} +{{- $tag := .Values.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "zookeeper.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if or .Values.image.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- else if or .Values.image.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "zookeeper.labels" -}} +app.kubernetes.io/name: {{ include "zookeeper.name" . }} +helm.sh/chart: {{ include "zookeeper.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "zookeeper.tplValue" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "zookeeper.tplValue" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "zookeeper.matchLabels" -}} +app.kubernetes.io/name: {{ include "zookeeper.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Return ZooKeeper Client Password +*/}} +{{- define "zookeeper.clientPassword" -}} +{{- if .Values.auth.clientPassword -}} + {{- .Values.auth.clientPassword -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return ZooKeeper Servers Passwords +*/}} +{{- define "zookeeper.serverPasswords" -}} +{{- if .Values.auth.serverPasswords -}} + {{- .Values.auth.serverPasswords -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "zookeeper.volumePermissions.image" -}} +{{- $registryName := .Values.volumePermissions.image.registry -}} +{{- $repositoryName := .Values.volumePermissions.image.repository -}} +{{- $tag := .Values.volumePermissions.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Storage Class +*/}} +{{- define "zookeeper.storageClass" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +*/}} +{{- if .Values.global -}} + {{- if .Values.global.storageClass -}} + {{- if (eq "-" .Values.global.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.global.storageClass -}} + {{- end -}} + {{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml new file mode 100755 index 000000000..1a4061565 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.config }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + zoo.cfg: |- +{{ .Values.config | indent 4 }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml new file mode 100755 index 000000000..3e26ed6c8 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml @@ -0,0 +1,29 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "zookeeper.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.metrics.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.metrics.service.annotations }} + {{ include "zookeeper.tplValue" ( dict "value" .Values.metrics.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type }} + ports: + - name: tcp-metrics + port: {{ .Values.metrics.service.port }} + targetPort: metrics + selector: {{- include "zookeeper.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml new file mode 100755 index 000000000..f7e30b4bc --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml @@ -0,0 +1,43 @@ +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ include "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + ingress: + # Allow inbound connections to zookeeper + - ports: + - port: {{ .Values.service.port }} + from: + {{- if not .Values.networkPolicy.allowExternal }} + - podSelector: + matchLabels: + {{ include "zookeeper.fullname" . }}-client: "true" + - podSelector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 14 }} + {{- else }} + - podSelector: + matchLabels: {} + {{- end }} + # Internal ports + - ports: &intranodes_ports + - port: {{ .Values.service.followerPort }} + - port: {{ .Values.service.electionPort }} + from: + - podSelector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 14 }} + egress: + - ports: *intranodes_ports + # Allow outbound connections from zookeeper nodes + +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..818950c66 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml @@ -0,0 +1,21 @@ +{{- $replicaCount := int .Values.replicaCount }} +{{- if gt $replicaCount 1 }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: zookeeper + {{- toYaml .Values.podDisruptionBudget | nindent 2 }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml new file mode 100755 index 000000000..9cda3985c --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled .Values.metrics.prometheusRule.rules }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "zookeeper.fullname" . }} + {{- if .Values.metrics.prometheusRule.namespace }} + namespace: {{ .Values.metrics.prometheusRule.namespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- range $key, $value := .Values.metrics.prometheusRule.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + groups: + - name: {{ include "zookeeper.fullname" . }} + rules: {{- toYaml .Values.metrics.prometheusRule.rules | nindent 6 }} +{{- end }} + diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml new file mode 100755 index 000000000..b3d727fec --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.auth.enabled (not .Values.auth.existingSecret) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + client-password: {{ include "zookeeper.clientPassword" . | b64enc | quote }} + server-password: {{ include "zookeeper.serverPasswords" . | b64enc | quote }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml new file mode 100755 index 000000000..3f7ef39fd --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "zookeeper.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + role: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml new file mode 100755 index 000000000..5782dad59 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "zookeeper.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- else }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: zookeeper + endpoints: + - port: tcp-metrics + path: "/metrics" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml new file mode 100755 index 000000000..fa1e5231f --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml @@ -0,0 +1,334 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + role: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + serviceName: {{ template "zookeeper.fullname" . }}-headless + replicas: {{ .Values.replicaCount }} + podManagementPolicy: {{ .Values.podManagementPolicy }} + updateStrategy: + type: {{ .Values.updateStrategy }} + {{- if (eq "Recreate" .Values.updateStrategy) }} + rollingUpdate: null + {{- else if .Values.rollingUpdatePartition }} + rollingUpdate: + partition: {{ .Values.rollingUpdatePartition }} + {{- end }} + selector: + matchLabels: {{- include "zookeeper.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: zookeeper + template: + metadata: + name: {{ template "zookeeper.fullname" . }} + labels: {{- include "zookeeper.labels" . | nindent 8 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.podLabels }} + {{- include "zookeeper.tplValue" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: {{- include "zookeeper.tplValue" ( dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + {{- include "zookeeper.imagePullSecrets" . | nindent 6 }} + serviceAccountName: {{ template "zookeeper.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "zookeeper.tplValue" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "zookeeper.tplValue" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "zookeeper.tplValue" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }} + initContainers: + - name: volume-permissions + image: {{ template "zookeeper.volumePermissions.image" . }} + imagePullPolicy: {{ default "" .Values.volumePermissions.image.pullPolicy | quote }} + command: + - chown + args: + - -R + - {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} + - /bitnami/zookeeper + {{- if .Values.dataLogDir }} + - {{ .Values.dataLogDir }} + {{- end }} + securityContext: + runAsUser: 0 + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + {{- if .Values.dataLogDir }} + - name: data-log + mountPath: {{ .Values.dataLogDir }} + {{- end }} + {{- end }} + containers: + - name: zookeeper + image: {{ template "zookeeper.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + command: + - bash + - -ec + - | + # Execute entrypoint as usual after obtaining ZOO_SERVER_ID based on POD hostname + HOSTNAME=`hostname -s` + if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then + ORD=${BASH_REMATCH[2]} + export ZOO_SERVER_ID=$((ORD+1)) + else + echo "Failed to get index from hostname $HOST" + exit 1 + fi + exec /entrypoint.sh /run.sh + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + env: + - name: ZOO_DATA_LOG_DIR + value: {{ .Values.dataLogDir | quote }} + - name: ZOO_PORT_NUMBER + value: {{ .Values.service.port | quote }} + - name: ZOO_TICK_TIME + value: {{ .Values.tickTime | quote }} + - name: ZOO_INIT_LIMIT + value: {{ .Values.initLimit | quote }} + - name: ZOO_SYNC_LIMIT + value: {{ .Values.syncLimit | quote }} + - name: ZOO_MAX_CLIENT_CNXNS + value: {{ .Values.maxClientCnxns | quote }} + - name: ZOO_4LW_COMMANDS_WHITELIST + value: {{ .Values.fourlwCommandsWhitelist | quote }} + - name: ZOO_LISTEN_ALLIPS_ENABLED + value: {{ ternary "yes" "no" .Values.listenOnAllIPs | quote }} + - name: ZOO_AUTOPURGE_INTERVAL + value: {{ .Values.autopurge.purgeInterval | quote }} + - name: ZOO_AUTOPURGE_RETAIN_COUNT + value: {{ .Values.autopurge.snapRetainCount | quote }} + - name: ZOO_MAX_SESSION_TIMEOUT + value: {{ .Values.maxSessionTimeout | quote }} + - name: ZOO_SERVERS + {{- $replicaCount := int .Values.replicaCount }} + {{- $followerPort := int .Values.service.followerPort }} + {{- $electionPort := int .Values.service.electionPort }} + {{- $releaseNamespace := .Release.Namespace }} + {{- $zookeeperFullname := include "zookeeper.fullname" . }} + {{- $zookeeperHeadlessServiceName := printf "%s-%s" $zookeeperFullname "headless" | trunc 63 }} + {{- $clusterDomain := .Values.clusterDomain }} + value: {{ range $i, $e := until $replicaCount }}{{ $zookeeperFullname }}-{{ $e }}.{{ $zookeeperHeadlessServiceName }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $followerPort }}:{{ $electionPort }} {{ end }} + - name: ZOO_ENABLE_AUTH + value: {{ ternary "yes" "no" .Values.auth.enabled | quote }} + {{- if .Values.auth.enabled }} + - name: ZOO_CLIENT_USER + value: {{ .Values.auth.clientUser | quote }} + - name: ZOO_CLIENT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.auth.existingSecret }}{{ .Values.auth.existingSecret }}{{ else }}{{ template "zookeeper.fullname" . }}{{ end }} + key: client-password + - name: ZOO_SERVER_USERS + value: {{ .Values.auth.serverUsers | quote }} + - name: ZOO_SERVER_PASSWORDS + valueFrom: + secretKeyRef: + name: {{ if .Values.auth.existingSecret }}{{ .Values.auth.existingSecret }}{{ else }}{{ template "zookeeper.fullname" . }}{{ end }} + key: server-password + {{- end }} + - name: ZOO_HEAP_SIZE + value: {{ .Values.heapSize | quote }} + - name: ZOO_LOG_LEVEL + value: {{ .Values.logLevel | quote }} + - name: ALLOW_ANONYMOUS_LOGIN + value: {{ ternary "yes" "no" .Values.allowAnonymousLogin | quote }} + {{- if .Values.jvmFlags }} + - name: JVMFLAGS + value: {{ .Values.jvmFlags | quote }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: ZOO_ENABLE_PROMETHEUS_METRICS + value: "yes" + - name: ZOO_PROMETHEUS_METRICS_PORT_NUMBER + value: {{ .Values.metrics.containerPort | quote }} + {{- end }} + {{- if .Values.service.tls.client_enable }} + - name: ZOO_TLS_CLIENT_ENABLE + value: {{ .Values.service.tls.client_enable | quote }} + - name: ZOO_TLS_CLIENT_KEYSTORE_FILE + value: {{ .Values.service.tls.client_keystore_path | quote }} + - name: ZOO_TLS_CLIENT_KEYSTORE_PASSWORD + value: {{ .Values.service.tls.client_keystore_password | quote }} + - name: ZOO_TLS_CLIENT_TRUSTSTORE_FILE + value: {{ .Values.service.tls.client_truststore_path | quote }} + - name: ZOO_TLS_CLIENT_TRUSTSTORE_PASSWORD + value: {{ .Values.service.tls.client_truststore_password | quote }} + {{ end }} + {{- if .Values.service.tls.quorum_enable }} + - name: ZOO_TLS_QUORUM_ENABLE + value: {{ .Values.service.tls.quorum_enable | quote }} + - name: ZOO_TLS_QUORUM_KEYSTORE_FILE + value: {{ .Values.service.tls.quorum_keystore_path | quote }} + - name: ZOO_TLS_QUORUM_KEYSTORE_PASSWORD + value: {{ .Values.service.tls.quorum_keystore_password | quote }} + - name: ZOO_TLS_QUORUM_TRUSTSTORE_FILE + value: {{ .Values.service.tls.quorum_truststore_path | quote }} + - name: ZOO_TLS_QUORUM_TRUSTSTORE_PASSWORD + value: {{ .Values.service.tls.quorum_truststore_password | quote }} + {{ end }} + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + {{- if .Values.extraEnvVars }} + {{- toYaml .Values.extraEnvVars | nindent 12 }} + {{- end }} + ports: + {{ if not .Values.service.tls.disable_base_client_port }} + - name: client + containerPort: {{ .Values.service.port }} + {{ end }} + {{ if .Values.service.tls.client_enable }} + - name: client-tls + containerPort: {{ .Values.service.tls.client_port }} + {{ end }} + - name: follower + containerPort: {{ .Values.service.followerPort }} + - name: election + containerPort: {{ .Values.service.electionPort }} + {{- if .Values.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.metrics.containerPort }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + {{- if not .Values.service.tls.disable_base_client_port }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.livenessProbe.probeCommandTimeout }} nc -w {{ .Values.livenessProbe.probeCommandTimeout }} localhost {{ .Values.service.port }} | grep imok'] + {{- else }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.livenessProbe.probeCommandTimeout }} openssl s_client -quiet -crlf -connect localhost:{{ .Values.service.tls.client_port }} | grep imok'] + {{- end }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + {{- if not .Values.service.tls.disable_base_client_port }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.readinessProbe.probeCommandTimeout }} nc -w {{ .Values.readinessProbe.probeCommandTimeout }} localhost {{ .Values.service.port }} | grep imok'] + {{- else }} + command: ['/bin/bash', '-c', 'echo "ruok" | timeout {{ .Values.readinessProbe.probeCommandTimeout }} openssl s_client -quiet -crlf -connect localhost:{{ .Values.service.tls.client_port }} | grep imok'] + {{- end }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + {{- if .Values.dataLogDir }} + - name: data-log + mountPath: {{ .Values.dataLogDir }} + {{- end }} + {{- if .Values.config }} + - name: config + mountPath: /opt/bitnami/zookeeper/conf/zoo.cfg + subPath: zoo.cfg + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + volumes: + {{- if .Values.config }} + - name: config + configMap: + name: {{ template "zookeeper.fullname" . }} + {{- end }} + {{- if and .Values.persistence.enabled .Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.persistence.existingClaim .) }} + {{- else if not .Values.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- if and .Values.persistence.enabled .Values.persistence.dataLogDir.existingClaim }} + - name: data-log + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.persistence.dataLogDir.existingClaim .) }} + {{- else if and ( not .Values.persistence.enabled ) .Values.dataLogDir }} + - name: data-log + emptyDir: {} + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if and .Values.persistence.enabled (not (and .Values.persistence.existingClaim .Values.persistence.dataLogDir.existingClaim) )}} + volumeClaimTemplates: + {{- if not .Values.persistence.existingClaim }} + - metadata: + name: data + annotations: + {{- range $key, $value := .Values.persistence.annotations }} + {{ $key }}: {{ $value }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- include "zookeeper.storageClass" . | nindent 8 }} + {{- end }} + {{- if and (not .Values.persistence.dataLogDir.existingClaim) .Values.dataLogDir }} + - metadata: + name: data-log + annotations: + {{- range $key, $value := .Values.persistence.annotations }} + {{ $key }}: {{ $value }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.dataLogDir.size | quote }} + {{- include "zookeeper.storageClass" . | nindent 8 }} + {{- end }} + {{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml new file mode 100755 index 000000000..972efb51d --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "zookeeper.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.commonAnnotations .Values.service.annotations }} + annotations: + {{- if .Values.service.headless.annotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.service.headless.annotations "context" $ ) | nindent 4 }}\ + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: {{ .Values.service.publishNotReadyAddresses }} + ports: + {{ if not .Values.service.tls.disable_base_client_port }} + - name: tcp-client + port: 2181 + targetPort: client + {{ end }} + {{ if .Values.service.tls.client_enable }} + - name: tcp-client-tls + port: {{ .Values.service.tls.client_port }} + targetPort: client-tls + {{ end }} + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: {{- include "zookeeper.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml new file mode 100755 index 000000000..da3a2895a --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "zookeeper.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "zookeeper.labels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper + {{- if .Values.commonLabels }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.commonAnnotations .Values.service.annotations }} + annotations: + {{- if .Values.service.annotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.service.annotations "context" $ ) | nindent 4 }}\ + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "zookeeper.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + {{ if not .Values.service.tls.disable_base_client_port }} + - name: tcp-client + port: 2181 + targetPort: client + {{ end }} + {{ if .Values.service.tls.client_enable }} + - name: tcp-client-tls + port: {{ .Values.service.tls.client_port }} + targetPort: client-tls + {{ end }} + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: {{- include "zookeeper.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: zookeeper diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml new file mode 100755 index 000000000..7d678603f --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml @@ -0,0 +1,430 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Zookeeper image version +## ref: https://hub.docker.com/r/bitnami/zookeeper/tags/ +## +image: + registry: docker.io + repository: bitnami/zookeeper + tag: 3.6.2-debian-10-r10 + + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + ## + debug: false + +## String to partially override zookeeper.fullname template (will maintain the release name) +# nameOverride: + +## String to fully override zookeeper.fullname template +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Example Use Cases: +## mount certificates to enable tls +# extraVolumes: +# - name: zookeeper-keystore +# secret: +# defaultMode: 288 +# secretName: zookeeper-keystore +# - name: zookeeper-trustsore +# secret: +# defaultMode: 288 +# secretName: zookeeper-truststore +# extraVolumeMounts: +# - name: zookeeper-keystore +# mountPath: /certs/keystore +# readOnly: true +# - name: zookeeper-truststore +# mountPath: /certs/truststore +# readOnly: true + + +## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Limits the number of pods of the replicated application that are down simultaneously from voluntary disruptions +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +podDisruptionBudget: + maxUnavailable: 1 + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy +## +podManagementPolicy: Parallel + +## Number of ZooKeeper nodes +## +replicaCount: 3 + +## Basic time unit in milliseconds used by ZooKeeper for heartbeats +## +tickTime: 2000 + +## ZooKeeper uses to limit the length of time the ZooKeeper servers in quorum have to connect to a leader +## +initLimit: 10 + +## How far out of date a server can be from a leader +## +syncLimit: 5 + +## Limits the number of concurrent connections that a single client may make to a single member of the ZooKeeper ensemble +## +maxClientCnxns: 60 + +## Maximum session timeout in milliseconds that the server will allow the client to negotiate. Defaults to 20 times the tickTime. +## +maxSessionTimeout: 40000 + +## A list of comma separated Four Letter Words commands to use +## +fourlwCommandsWhitelist: srvr, mntr, ruok + +## Allow zookeeper to listen for peers on all IPs +## +listenOnAllIPs: false + +## Allow to accept connections from unauthenticated users +## +allowAnonymousLogin: true + +autopurge: + ## Retains the snapRetainCount most recent snapshots and the corresponding transaction logs and deletes the rest + ## + snapRetainCount: 3 + ## The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. + ## + purgeInterval: 0 + +auth: + ## Use existing secret (ignores previous password) + ## + # existingSecret: + ## Enable Zookeeper auth. It uses SASL/Digest-MD5 + ## + enabled: false + ## User that will use Zookeeper clients to auth + ## + clientUser: + ## Password that will use Zookeeper clients to auth + ## + clientPassword: + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + serverUsers: + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + serverPasswords: + +## Size in MB for the Java Heap options (Xmx and XMs). This env var is ignored if Xmx an Xms are configured via JVMFLAGS +## +heapSize: 1024 + +## Log level for the Zookeeper server. ERROR by default. Have in mind if you set it to INFO or WARN the ReadinessProve will produce a lot of logs. +## +logLevel: ERROR + +## Data log directory. Specifying this option will direct zookeeper to write the transaction log to the dataLogDir rather than the dataDir. +## This allows a dedicated log device to be used, and helps avoid competition between logging and snaphots. +## Example: +## dataLogDir: /bitnami/zookeeper/dataLog +## +dataLogDir: "" + +## Default JVMFLAGS for the ZooKeeper process +## +# jvmFlags: + +## Configure ZooKeeper with a custom zoo.cfg file +## +# config: + +## Kubernetes configuration +## For minikube, set this to NodePort, elsewhere use LoadBalancer +## +service: + type: ClusterIP + port: 2181 + followerPort: 2888 + electionPort: 3888 + publishNotReadyAddresses: true + tls: + client_enable: true + quorum_enable: true + disable_base_client_port: true + + client_port: 3181 + + client_keystore_path: /tls_key_store/key_store_file + client_keystore_password: "" + client_truststore_path: /tls_trust_store/trust_store_file + client_truststore_password: "" + + quorum_keystore_path: /tls_key_store/key_store_file + quorum_keystore_password: "" + quorum_truststore_path: /tls_trust_store/trust_store_file + quorum_truststore_password: "" + annotations: {} + headless: + annotations: {} + +## Service account for Zookeeper to use. +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the zookeeper.fullname template + # name: + +## Zookeeper Pod Security Context +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Zookeeper data Persistent Volume Storage Class +## If defined, storageClassName: +## If set to "-", storageClassName: "", which disables dynamic provisioning +## If undefined (the default) or set to null, no storageClassName spec is +## set, choosing the default provisioner. (gp2 on AWS, standard on +## GKE, AWS & OpenStack) +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + dataLogDir: + size: 8Gi + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Labels +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +## Annotations +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +## Name of the priority class to be used by zookeeper pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: "" + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Scheduler name +## https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: stork + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 256Mi + cpu: 250m + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +## Network policies +## Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ +## +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: true + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis is listening + ## on. When true, zookeeper accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + +## Zookeeper Prometheus Exporter configuration +## +metrics: + enabled: false + + ## Zookeeper Prometheus Exporter container port + ## + containerPort: 9141 + + ## Service configuration + ## + service: + ## Zookeeper Prometheus Exporter service type + ## + type: ClusterIP + ## Zookeeper Prometheus Exporter service port + ## + port: 9141 + ## Annotations for the Zookeeper Prometheus Exporter metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace for the ServiceMonitor Resource (defaults to the Release Namespace) + ## + namespace: + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Prometheus Operator PrometheusRule configuration + ## + prometheusRule: + enabled: false + ## Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: + + ## PrometheusRule selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Some example rules. + rules: [] + # - alert: ZookeeperSyncedFollowers + # annotations: + # message: The number of synced followers for the leader node in Zookeeper deployment my-release is less than 2. This usually means that some of the Zookeeper nodes aren't communicating properly. If it doesn't resolve itself you can try killing the pods (one by one). + # expr: max(synced_followers{service="my-release-metrics"}) < 2 + # for: 5m + # labels: + # severity: critical + # - alert: ZookeeperOutstandingRequests + # annotations: + # message: The number of outstanding requests for Zookeeper pod {{ $labels.pod }} is greater than 10. This can indicate a performance issue with the Pod or cluster a whole. + # expr: outstanding_requests{service="my-release-metrics"} > 10 + # for: 5m + # labels: + # severity: critical diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml new file mode 100755 index 000000000..a40decb54 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml @@ -0,0 +1,430 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Zookeeper image version +## ref: https://hub.docker.com/r/bitnami/zookeeper/tags/ +## +image: + registry: docker.io + repository: bitnami/zookeeper + tag: 3.6.2-debian-10-r10 + + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + ## + debug: false + +## String to partially override zookeeper.fullname template (will maintain the release name) +# nameOverride: + +## String to fully override zookeeper.fullname template +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Example Use Cases: +## mount certificates to enable tls +# extraVolumes: +# - name: zookeeper-keystore +# secret: +# defaultMode: 288 +# secretName: zookeeper-keystore +# - name: zookeeper-trustsore +# secret: +# defaultMode: 288 +# secretName: zookeeper-truststore +# extraVolumeMounts: +# - name: zookeeper-keystore +# mountPath: /certs/keystore +# readOnly: true +# - name: zookeeper-truststore +# mountPath: /certs/truststore +# readOnly: true + +## StatefulSet controller supports automated updates. There are two valid update strategies: RollingUpdate and OnDelete +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Limits the number of pods of the replicated application that are down simultaneously from voluntary disruptions +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +podDisruptionBudget: + maxUnavailable: 1 + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy +## +podManagementPolicy: Parallel + +## Number of ZooKeeper nodes +## +replicaCount: 1 + +## Basic time unit in milliseconds used by ZooKeeper for heartbeats +## +tickTime: 2000 + +## ZooKeeper uses to limit the length of time the ZooKeeper servers in quorum have to connect to a leader +## +initLimit: 10 + +## How far out of date a server can be from a leader +## +syncLimit: 5 + +## Limits the number of concurrent connections that a single client may make to a single member of the ZooKeeper ensemble +## +maxClientCnxns: 60 + +## A list of comma separated Four Letter Words commands to use +## +fourlwCommandsWhitelist: srvr, mntr, ruok + +## Allow zookeeper to listen for peers on all IPs +## +listenOnAllIPs: false + +## Allow to accept connections from unauthenticated users +## +allowAnonymousLogin: true + +autopurge: + ## Retains the snapRetainCount most recent snapshots and the corresponding transaction logs and deletes the rest + ## + snapRetainCount: 3 + ## The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. + ## + purgeInterval: 0 + +## Maximum session timeout in milliseconds that the server will allow the client to negotiate. Defaults to 20 times the tickTime. +## +maxSessionTimeout: 40000 + +auth: + ## Use existing secret (ignores previous password) + ## + # existingSecret: + ## Enable Zookeeper auth. It uses SASL/Digest-MD5 + ## + enabled: false + ## User that will use Zookeeper clients to auth + ## + clientUser: + ## Password that will use Zookeeper clients to auth + ## + clientPassword: + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + serverUsers: + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + serverPasswords: + +## Size in MB for the Java Heap options (Xmx and XMs). This env var is ignored if Xmx an Xms are configured via JVMFLAGS +## +heapSize: 1024 + +## Log level for the Zookeeper server. ERROR by default. Have in mind if you set it to INFO or WARN the ReadinessProve will produce a lot of logs. +## +logLevel: ERROR + +## Data log directory. Specifying this option will direct zookeeper to write the transaction log to the dataLogDir rather than the dataDir. +## This allows a dedicated log device to be used, and helps avoid competition between logging and snaphots. +## Example: +## dataLogDir: /bitnami/zookeeper/dataLog +## +dataLogDir: "" + +## Default JVMFLAGS for the ZooKeeper process +## +# jvmFlags: + +## Configure ZooKeeper with a custom zoo.cfg file +## +# config: + +## Kubernetes configuration +## For minikube, set this to NodePort, elsewhere use LoadBalancer +## +service: + type: ClusterIP + port: 2181 + followerPort: 2888 + electionPort: 3888 + publishNotReadyAddresses: true + tls: + client_enable: false + quorum_enable: false + disable_base_client_port: false + + client_port: 3181 + + client_keystore_path: /tls_key_store/key_store_file + client_keystore_password: "" + client_truststore_path: /tls_trust_store/trust_store_file + client_truststore_password: "" + + quorum_keystore_path: /tls_key_store/key_store_file + quorum_keystore_password: "" + quorum_truststore_path: /tls_trust_store/trust_store_file + quorum_truststore_password: "" + annotations: {} + headless: + annotations: {} + +## Service account for Zookeeper to use. +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the zookeeper.fullname template + # name: + +## Zookeeper Pod Security Context +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Zookeeper data Persistent Volume Storage Class +## If defined, storageClassName: +## If set to "-", storageClassName: "", which disables dynamic provisioning +## If undefined (the default) or set to null, no storageClassName spec is +## set, choosing the default provisioner. (gp2 on AWS, standard on +## GKE, AWS & OpenStack) +## +persistence: + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + + enabled: true + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + dataLogDir: + size: 8Gi + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Labels +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +## Annotations +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +## Name of the priority class to be used by zookeeper pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: "" + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Scheduler name +## https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: stork + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 256Mi + cpu: 250m + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + probeCommandTimeout: 2 + +## Network policies +## Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ +## +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis is listening + ## on. When true, zookeeper accept connections from any source + ## (with the correct destination port). + ## + # allowExternal: true + +## Zookeeper Prometheus Exporter configuration +## +metrics: + enabled: false + + ## Zookeeper Prometheus Exporter container port + ## + containerPort: 9141 + + ## Service configuration + ## + service: + ## Zookeeper Prometheus Exporter service type + ## + type: ClusterIP + ## Zookeeper Prometheus Exporter service port + ## + port: 9141 + ## Annotations for the Zookeeper Prometheus Exporter metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace for the ServiceMonitor Resource (defaults to the Release Namespace) + ## + namespace: + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Prometheus Operator PrometheusRule configuration + ## + prometheusRule: + enabled: false + ## Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: + + ## PrometheusRule selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + + ## Some example rules. + rules: [] + # - alert: ZookeeperSyncedFollowers + # annotations: + # message: The number of synced followers for the leader node in Zookeeper deployment my-release is less than 2. This usually means that some of the Zookeeper nodes aren't communicating properly. If it doesn't resolve itself you can try killing the pods (one by one). + # expr: max(synced_followers{service="my-release-metrics"}) < 2 + # for: 5m + # labels: + # severity: critical + # - alert: ZookeeperOutstandingRequests + # annotations: + # message: The number of outstanding requests for Zookeeper pod {{ $labels.pod }} is greater than 10. This can indicate a performance issue with the Pod or cluster a whole. + # expr: outstanding_requests{service="my-release-metrics"} > 10 + # for: 5m + # labels: + # severity: critical diff --git a/scripts/helm/helm/databases/charts/kafka/files/jks/README.md b/scripts/helm/helm/databases/charts/kafka/files/jks/README.md new file mode 100755 index 000000000..e110a8825 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/files/jks/README.md @@ -0,0 +1,10 @@ +# Java Key Stores + +You can copy here your Java Key Stores (JKS) files so a secret is created including them. Remember to use a truststore (`kafka.truststore.jks`) and one keystore (`kafka.keystore.jks`) per Kafka broker you have in the cluster. For instance, if you have 3 brokers you need to copy here the following files: + +- kafka.truststore.jks +- kafka-0.keystore.jks +- kafka-1.keystore.jks +- kafka-2.keystore.jks + +Find more info in [this section](https://github.com/bitnami/charts/tree/master/bitnami/kafka#enable-security-for-kafka-and-zookeeper) of the README.md file. diff --git a/scripts/helm/helm/databases/charts/kafka/kafka.yaml b/scripts/helm/helm/databases/charts/kafka/kafka.yaml new file mode 100644 index 000000000..acd718957 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/kafka.yaml @@ -0,0 +1,521 @@ +--- +# Source: kafka/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kafka + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +--- +# Source: kafka/templates/scripts-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-scripts + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm +data: + setup.sh: |- + #!/bin/bash + + ID="${MY_POD_NAME#"kafka-"}" + export KAFKA_CFG_BROKER_ID="$ID" + + exec /entrypoint.sh /run.sh +--- +# Source: kafka/charts/zookeeper/templates/svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-zookeeper-headless + namespace: db + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + + - name: tcp-client + port: 2181 + targetPort: client + + + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: zookeeper +--- +# Source: kafka/charts/zookeeper/templates/svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-zookeeper + namespace: db + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper +spec: + type: ClusterIP + ports: + + - name: tcp-client + port: 2181 + targetPort: client + + + - name: follower + port: 2888 + targetPort: follower + - name: tcp-election + port: 3888 + targetPort: election + selector: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: zookeeper +--- +# Source: kafka/templates/kafka-metrics-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-metrics + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + annotations: + + prometheus.io/path: /metrics + prometheus.io/port: '9308' + prometheus.io/scrape: "true" +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 9308 + protocol: TCP + targetPort: metrics + nodePort: null + selector: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: metrics +--- +# Source: kafka/templates/svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka-headless + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-client + port: 9092 + protocol: TCP + targetPort: kafka-client + - name: tcp-internal + port: 9093 + protocol: TCP + targetPort: kafka-internal + selector: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: kafka +--- +# Source: kafka/templates/svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: kafka + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +spec: + type: ClusterIP + ports: + - name: tcp-client + port: 9092 + protocol: TCP + targetPort: kafka-client + nodePort: null + selector: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: kafka +--- +# Source: kafka/templates/kafka-metrics-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-exporter + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: metrics + template: + metadata: + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + spec: + containers: + - name: kafka-exporter + image: docker.io/bitnami/kafka-exporter:1.2.0-debian-10-r220 + imagePullPolicy: "IfNotPresent" + command: + - /bin/bash + - -ec + - | + read -r -a sasl_passwords <<< "$(tr ',;' ' ' <<< "${SASL_USER_PASSWORD}")" + kafka_exporter \ + --kafka.server=kafka-0.kafka-headless.db.svc.cluster.local:9092 \ + --kafka.server=kafka-1.kafka-headless.db.svc.cluster.local:9092 \ + --web.listen-address=:9308 + ports: + - name: metrics + containerPort: 9308 + resources: + limits: {} + requests: {} +--- +# Source: kafka/charts/zookeeper/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka-zookeeper + namespace: db + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper + role: zookeeper +spec: + serviceName: kafka-zookeeper-headless + replicas: 1 + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: zookeeper + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: zookeeper + template: + metadata: + name: kafka-zookeeper + labels: + app.kubernetes.io/name: zookeeper + helm.sh/chart: zookeeper-5.21.9 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: zookeeper + spec: + + serviceAccountName: default + securityContext: + fsGroup: 1001 + containers: + - name: zookeeper + image: docker.io/bitnami/zookeeper:3.6.2-debian-10-r10 + imagePullPolicy: "IfNotPresent" + securityContext: + runAsUser: 1001 + command: + - bash + - -ec + - | + # Execute entrypoint as usual after obtaining ZOO_SERVER_ID based on POD hostname + HOSTNAME=`hostname -s` + if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then + ORD=${BASH_REMATCH[2]} + export ZOO_SERVER_ID=$((ORD+1)) + else + echo "Failed to get index from hostname $HOST" + exit 1 + fi + exec /entrypoint.sh /run.sh + resources: + requests: + cpu: 250m + memory: 256Mi + env: + - name: ZOO_DATA_LOG_DIR + value: "" + - name: ZOO_PORT_NUMBER + value: "2181" + - name: ZOO_TICK_TIME + value: "2000" + - name: ZOO_INIT_LIMIT + value: "10" + - name: ZOO_SYNC_LIMIT + value: "5" + - name: ZOO_MAX_CLIENT_CNXNS + value: "60" + - name: ZOO_4LW_COMMANDS_WHITELIST + value: "srvr, mntr, ruok" + - name: ZOO_LISTEN_ALLIPS_ENABLED + value: "no" + - name: ZOO_AUTOPURGE_INTERVAL + value: "0" + - name: ZOO_AUTOPURGE_RETAIN_COUNT + value: "3" + - name: ZOO_MAX_SESSION_TIMEOUT + value: "40000" + - name: ZOO_SERVERS + value: kafka-zookeeper-0.kafka-zookeeper-headless.db.svc.cluster.local:2888:3888 + - name: ZOO_ENABLE_AUTH + value: "no" + - name: ZOO_HEAP_SIZE + value: "1024" + - name: ZOO_LOG_LEVEL + value: "ERROR" + - name: ALLOW_ANONYMOUS_LOGIN + value: "yes" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + ports: + + - name: client + containerPort: 2181 + + + - name: follower + containerPort: 2888 + - name: election + containerPort: 3888 + livenessProbe: + exec: + command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok'] + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + exec: + command: ['/bin/bash', '-c', 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok'] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + volumeMounts: + - name: data + mountPath: /bitnami/zookeeper + volumes: + volumeClaimTemplates: + - metadata: + name: data + annotations: + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" +--- +# Source: kafka/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka +spec: + podManagementPolicy: Parallel + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: kafka + app.kubernetes.io/instance: kafka + app.kubernetes.io/component: kafka + serviceName: kafka-headless + updateStrategy: + type: "RollingUpdate" + template: + metadata: + labels: + app.kubernetes.io/name: kafka + helm.sh/chart: kafka-11.8.6 + app.kubernetes.io/instance: kafka + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: kafka + spec: + securityContext: + fsGroup: 1001 + runAsUser: 1001 + serviceAccountName: kafka + containers: + - name: kafka + image: docker.io/bitnami/kafka:2.6.0-debian-10-r30 + imagePullPolicy: "IfNotPresent" + command: + - /scripts/setup.sh + env: + - name: BITNAMI_DEBUG + value: "false" + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KAFKA_CFG_ZOOKEEPER_CONNECT + value: "kafka-zookeeper" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: "INTERNAL" + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: "INTERNAL:PLAINTEXT,CLIENT:PLAINTEXT" + - name: KAFKA_CFG_LISTENERS + value: "INTERNAL://:9093,CLIENT://:9092" + - name: KAFKA_CFG_ADVERTISED_LISTENERS + value: "INTERNAL://$(MY_POD_NAME).kafka-headless.db.svc.cluster.local:9093,CLIENT://$(MY_POD_NAME).kafka-headless.db.svc.cluster.local:9092" + - name: ALLOW_PLAINTEXT_LISTENER + value: "yes" + - name: KAFKA_CFG_DELETE_TOPIC_ENABLE + value: "false" + - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE + value: "true" + - name: KAFKA_HEAP_OPTS + value: "-Xmx1024m -Xms1024m" + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES + value: "10000" + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS + value: "1000" + - name: KAFKA_CFG_LOG_RETENTION_BYTES + value: "1073741824" + - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS + value: "300000" + - name: KAFKA_CFG_LOG_RETENTION_HOURS + value: "168" + - name: KAFKA_CFG_MESSAGE_MAX_BYTES + value: "1000012" + - name: KAFKA_CFG_LOG_SEGMENT_BYTES + value: "1073741824" + - name: KAFKA_CFG_LOG_DIRS + value: "/bitnami/kafka/data" + - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR + value: "1" + - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: "1" + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR + value: "1" + - name: KAFKA_CFG_NUM_IO_THREADS + value: "8" + - name: KAFKA_CFG_NUM_NETWORK_THREADS + value: "3" + - name: KAFKA_CFG_NUM_PARTITIONS + value: "1" + - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR + value: "1" + - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES + value: "102400" + - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES + value: "104857600" + - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES + value: "102400" + - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS + value: "6000" + ports: + - name: kafka-client + containerPort: 9092 + - name: kafka-internal + containerPort: 9093 + livenessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 10 + timeoutSeconds: 5 + failureThreshold: + periodSeconds: + successThreshold: + readinessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + periodSeconds: + successThreshold: + resources: + limits: {} + requests: {} + volumeMounts: + - name: data + mountPath: /bitnami/kafka + - name: scripts + mountPath: /scripts/setup.sh + subPath: setup.sh + volumes: + - name: scripts + configMap: + name: kafka-scripts + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" diff --git a/scripts/helm/helm/databases/charts/kafka/requirements.lock b/scripts/helm/helm/databases/charts/kafka/requirements.lock new file mode 100755 index 000000000..115d0b229 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: zookeeper + repository: https://charts.bitnami.com/bitnami + version: 5.21.9 +digest: sha256:2f3c43ce02e3966648b8c89be121fe39537f62ea1d161ad908f51ddc90e4243e +generated: "2020-09-29T07:43:56.483358254Z" diff --git a/scripts/helm/helm/databases/charts/kafka/requirements.yaml b/scripts/helm/helm/databases/charts/kafka/requirements.yaml new file mode 100755 index 000000000..533875258 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: + - name: zookeeper + version: 5.x.x + repository: https://charts.bitnami.com/bitnami + condition: zookeeper.enabled diff --git a/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt b/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt new file mode 100755 index 000000000..0347c21c4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt @@ -0,0 +1,181 @@ +{{- $replicaCount := int .Values.replicaCount -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $clusterDomain := .Values.clusterDomain -}} +{{- $fullname := include "kafka.fullname" . -}} +{{- $clientProtocol := include "kafka.listenerType" ( dict "protocol" .Values.auth.clientProtocol ) -}} +{{- $servicePort := int .Values.service.port -}} +{{- $loadBalancerIPListLength := len .Values.externalAccess.service.loadBalancerIPs -}} +{{- if and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $loadBalancerIPListLength )) (eq .Values.externalAccess.service.type "LoadBalancer") }} + +############################################################################### +### ERROR: You enabled external access to Kafka brokers without specifying ### +### the array of load balancer IPs for Kafka brokers. ### +############################################################################### + +This deployment will be incomplete until you configure the array of load balancer +IPs for Kafka brokers. To complete your deployment follow the steps below: + +1. Wait for the load balancer IPs (it may take a few minutes for them to be available): + + kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -w + +2. Obtain the load balancer IPs and upgrade your chart: + + {{- range $i, $e := until $replicaCount }} + LOAD_BALANCER_IP_{{ add $i 1 }}="$(kubectl get svc --namespace {{ $releaseNamespace }} {{ $fullname }}-{{ $i }}-external -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" + {{- end }} + +3. Upgrade you chart: + + helm upgrade {{ .Release.Name }} bitnami/{{ .Chart.Name }} \ + --set replicaCount={{ $replicaCount }} \ + --set externalAccess.enabled=true \ + {{- range $i, $e := until $replicaCount }} + --set externalAccess.service.loadBalancerIPs[{{ $i }}]=$LOAD_BALANCER_IP_{{ add $i 1 }} \ + {{- end }} + --set externalAccess.service.type=LoadBalancer + +{{- else }} + +{{- if and (or (eq .Values.service.type "LoadBalancer") .Values.externalAccess.enabled) (eq .Values.auth.clientProtocol "plaintext") }} +--------------------------------------------------------------------------------------------- + WARNING + + By specifying "serviceType=LoadBalancer" and not configuring the authentication + you have most likely exposed the Kafka service externally without any + authentication mechanism. + + For security reasons, we strongly suggest that you switch to "ClusterIP" or + "NodePort". As alternative, you can also configure the Kafka authentication. + +--------------------------------------------------------------------------------------------- +{{- end }} + +** Please be patient while the chart is being deployed ** + +Kafka can be accessed by consumers via port {{ $servicePort }} on the following DNS name from within your cluster: + + {{ $fullname }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }} + +Each Kafka broker can be accessed by producers via port {{ $servicePort }} on the following DNS name(s) from within your cluster: + +{{- $brokerList := list }} +{{- range $e, $i := until $replicaCount }} +{{- $brokerList = append $brokerList (printf "%s-%d.%s-headless.%s.svc.%s:%d" $fullname $i $fullname $releaseNamespace $clusterDomain $servicePort) }} +{{- end }} +{{ join "\n" $brokerList | nindent 4 }} + + +{{- if (include "kafka.client.saslAuthentication" .) }} + +You need to configure your Kafka client to access using SASL authentication. To do so, you need to create the 'kafka_jaas.conf' and 'client.properties' configuration files by executing these commands: + + - kafka_jaas.conf: + +cat > kafka_jaas.conf < client.properties <<>(Value) + name: kafka_controller_$1_$2_$4 + labels: + broker_id: "$3" + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.controller<>(Value) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.controller<>(Count) + name: kafka_controller_$1_$2_$3 + - pattern: kafka.server<>(Value) + name: kafka_server_$1_$2_$4 + labels: + client_id: "$3" + - pattern : kafka.network<>(Value) + name: kafka_network_$1_$2_$4 + labels: + network_processor: $3 + - pattern : kafka.network<>(Count) + name: kafka_network_$1_$2_$4 + labels: + request: $3 + - pattern: kafka.server<>(Count|OneMinuteRate) + name: kafka_server_$1_$2_$4 + labels: + topic: $3 + - pattern: kafka.server<>(Value) + name: kafka_server_$1_$2_$3_$4 + - pattern: kafka.server<>(Count|Value|OneMinuteRate) + name: kafka_server_$1_total_$2_$3 + - pattern: kafka.server<>(queue-size) + name: kafka_server_$1_$2 + - pattern: java.lang<(.+)>(\w+) + name: java_lang_$1_$4_$3_$2 + - pattern: java.lang<>(\w+) + name: java_lang_$1_$3_$2 + - pattern : java.lang + - pattern: kafka.log<>Value + name: kafka_log_$1_$2 + labels: + topic: $3 + partition: $4 +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml b/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml new file mode 100755 index 000000000..83edd8422 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml @@ -0,0 +1,45 @@ +{{- if .Values.metrics.jmx.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }}-jmx-metrics + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.metrics.jmx.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.metrics.jmx.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" .Values.metrics.jmx.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.jmx.service.type }} + {{- if eq .Values.metrics.jmx.service.type "LoadBalancer" }} + {{- if .Values.metrics.jmx.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.metrics.jmx.service.loadBalancerIP }} + {{- end }} + {{- if .Values.metrics.jmx.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.metrics.jmx.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + {{- if and (eq .Values.metrics.jmx.service.type "ClusterIP") .Values.metrics.jmx.service.clusterIP }} + clusterIP: {{ .Values.metrics.jmx.service.clusterIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.metrics.jmx.service.port }} + protocol: TCP + targetPort: metrics + {{- if and (or (eq .Values.metrics.jmx.service.type "NodePort") (eq .Values.metrics.jmx.service.type "LoadBalancer")) (not (empty .Values.metrics.jmx.service.nodePort)) }} + nodePort: {{ .Values.metrics.jmx.service.nodePort }} + {{- else if eq .Values.metrics.jmx.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: kafka +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml b/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml new file mode 100755 index 000000000..c547fbb39 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml @@ -0,0 +1,87 @@ +{{- if .Values.metrics.kafka.enabled }} +{{- $replicaCount := int .Values.replicaCount -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $clusterDomain := .Values.clusterDomain -}} +{{- $fullname := include "kafka.fullname" . -}} +{{- $servicePort := int .Values.service.port -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kafka.fullname" . }}-exporter + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: 1 + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: metrics + template: + metadata: + labels: {{- include "kafka.labels" . | nindent 8 }} + app.kubernetes.io/component: metrics + spec: +{{- include "kafka.imagePullSecrets" . | indent 6 }} + containers: + - name: kafka-exporter + image: {{ include "kafka.metrics.kafka.image" . }} + imagePullPolicy: {{ .Values.metrics.kafka.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + read -r -a sasl_passwords <<< "$(tr ',;' ' ' <<< "${SASL_USER_PASSWORD}")" + kafka_exporter \ + {{- range $i, $e := until $replicaCount }} + --kafka.server={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $servicePort }} \ + {{- end }} + {{- if (include "kafka.client.saslAuthentication" .) }} + --sasl.enabled \ + --sasl.username="$SASL_USERNAME" \ + --sasl.password="${sasl_passwords[0]}" \ + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + --tls.enabled \ + {{- if .Values.metrics.kafka.certificatesSecret }} + --tls.ca-file="/opt/bitnami/kafka-exporter/certs/ca-file" \ + --tls.cert-file="/opt/bitnami/kafka-exporter/certs/cert-file" \ + --tls.key-file="/opt/bitnami/kafka-exporter/certs/key-file" \ + {{- end }} + {{- end }} + {{- range $key, $value := .Values.metrics.kafka.extraFlags }} + --{{ $key }}{{ if $value }}={{ $value }}{{ end }} \ + {{- end }} + --web.listen-address=:9308 + {{- if (include "kafka.client.saslAuthentication" .) }} + env: + - name: SASL_USERNAME + value: {{ index .Values.auth.jaas.clientUsers 0 | quote }} + - name: SASL_USER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: client-passwords + {{- end }} + ports: + - name: metrics + containerPort: 9308 + {{- if .Values.metrics.kafka.resources }} + resources: {{ toYaml .Values.metrics.kafka.resources | nindent 12 }} + {{- end }} + {{- if and (include "kafka.tlsEncryption" .) .Values.metrics.kafka.certificatesSecret }} + volumeMounts: + - name: kafka-exporter-certificates + mountPath: /opt/bitnami/kafka-exporter/certs/ + readOnly: true + volumes: + - name: kafka-exporter-certificates + secret: + secretName: {{ .Values.metrics.kafka.certificatesSecret }} + defaultMode: 0440 + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml b/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml new file mode 100755 index 000000000..54a4ccb0b --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml @@ -0,0 +1,45 @@ +{{- if .Values.metrics.kafka.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }}-metrics + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.metrics.kafka.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.metrics.kafka.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" .Values.metrics.kafka.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.metrics.kafka.service.type }} + {{- if eq .Values.metrics.kafka.service.type "LoadBalancer" }} + {{- if .Values.metrics.kafka.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.metrics.kafka.service.loadBalancerIP }} + {{- end }} + {{- if .Values.metrics.kafka.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.metrics.kafka.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + {{- if and (eq .Values.metrics.kafka.service.type "ClusterIP") .Values.metrics.kafka.service.clusterIP }} + clusterIP: {{ .Values.metrics.kafka.service.clusterIP }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.metrics.kafka.service.port }} + protocol: TCP + targetPort: metrics + {{- if and (or (eq .Values.metrics.kafka.service.type "NodePort") (eq .Values.metrics.kafka.service.type "LoadBalancer")) (not (empty .Values.metrics.kafka.service.nodePort)) }} + nodePort: {{ .Values.metrics.kafka.service.nodePort }} + {{- else if eq .Values.metrics.kafka.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: metrics +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml b/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml new file mode 100755 index 000000000..0a34d50dd --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml @@ -0,0 +1,16 @@ +{{- if (include "kafka.log4j.createConfigMap" .) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kafka.log4j.configMapName" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + {{ .Values.log4j | nindent 4 }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml b/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..cf515becb --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml @@ -0,0 +1,25 @@ +{{- $replicaCount := int .Values.replicaCount }} +{{- if and .Values.pdb.create (gt $replicaCount 1) }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- if .Values.pdb.minAvailable }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{- end }} + {{- if .Values.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: kafka +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/role.yaml b/scripts/helm/helm/databases/charts/kafka/templates/role.yaml new file mode 100755 index 000000000..943c5bf3c --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/role.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +rules: + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml b/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml new file mode 100755 index 000000000..78f940f85 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.serviceAccount.create .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +roleRef: + kind: Role + name: {{ template "kafka.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ template "kafka.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml b/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml new file mode 100755 index 000000000..705545a61 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kafka.fullname" . }}-scripts + labels: {{- include "kafka.labels" . | nindent 4 }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + {{- $fullname := include "kafka.fullname" . }} + {{- $releaseNamespace := .Release.Namespace }} + {{- $clusterDomain := .Values.clusterDomain }} + {{- $interBrokerPort := .Values.service.internalPort }} + {{- $clientPort := .Values.service.port }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + auto-discovery.sh: |- + #!/bin/bash + + SVC_NAME="${MY_POD_NAME}-external" + + {{- if eq .Values.externalAccess.service.type "LoadBalancer" }} + # Auxiliar functions + retry_while() { + local -r cmd="${1:?cmd is missing}" + local -r retries="${2:-12}" + local -r sleep_time="${3:-5}" + local return_value=1 + + read -r -a command <<< "$cmd" + for ((i = 1 ; i <= retries ; i+=1 )); do + "${command[@]}" && return_value=0 && break + sleep "$sleep_time" + done + return $return_value + } + k8s_svc_lb_ip() { + local namespace=${1:?namespace is missing} + local service=${2:?service is missing} + local service_ip=$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.status.loadBalancer.ingress[0].ip}") + local service_hostname=$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}") + + if [[ -n ${service_ip} ]]; then + echo "${service_ip}" + else + echo "${service_hostname}" + fi + } + k8s_svc_lb_ip_ready() { + local namespace=${1:?namespace is missing} + local service=${2:?service is missing} + [[ -n "$(k8s_svc_lb_ip "$namespace" "$service")" ]] + } + # Wait until LoadBalancer IP is ready + retry_while "k8s_svc_lb_ip_ready {{ $releaseNamespace }} $SVC_NAME" || exit 1 + # Obtain LoadBalancer external IP + k8s_svc_lb_ip "{{ $releaseNamespace }}" "$SVC_NAME" | tee "$SHARED_FILE" + {{- else if eq .Values.externalAccess.service.type "NodePort" }} + k8s_svc_node_port() { + local namespace=${1:?namespace is missing} + local service=${2:?service is missing} + local index=${3:-0} + local node_port="$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.spec.ports[$index].nodePort}")" + echo "$node_port" + } + k8s_svc_node_port "{{ $releaseNamespace }}" "$SVC_NAME" | tee "$SHARED_FILE" + {{- end }} + {{- end }} + setup.sh: |- + #!/bin/bash + + ID="${MY_POD_NAME#"{{ $fullname }}-"}" + export KAFKA_CFG_BROKER_ID="$ID" + + {{- if .Values.externalAccess.enabled }} + # Configure external ip and port + {{- if eq .Values.externalAccess.service.type "LoadBalancer" }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + export EXTERNAL_ACCESS_IP="$(<${SHARED_FILE})" + {{- else }} + export EXTERNAL_ACCESS_IP=$(echo '{{ .Values.externalAccess.service.loadBalancerIPs }}' | tr -d '[]' | cut -d ' ' -f "$(($ID + 1))") + {{- end }} + export EXTERNAL_ACCESS_PORT={{ .Values.externalAccess.service.port }} + {{- else if eq .Values.externalAccess.service.type "NodePort" }} + {{- if .Values.externalAccess.service.domain }} + export EXTERNAL_ACCESS_IP={{ .Values.externalAccess.service.domain }} + {{- else }} + export EXTERNAL_ACCESS_IP=$(curl -s https://ipinfo.io/ip) + {{- end }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + export EXTERNAL_ACCESS_PORT="$(<${SHARED_FILE})" + {{- else }} + export EXTERNAL_ACCESS_PORT=$(echo '{{ .Values.externalAccess.service.nodePorts }}' | tr -d '[]' | cut -d ' ' -f "$(($ID + 1))") + {{- end }} + {{- end }} + + # Configure Kafka advertised listeners + {{- if .Values.advertisedListeners }} + export KAFKA_CFG_ADVERTISED_LISTENERS={{ .Values.advertisedListeners }} + {{- else }} + export KAFKA_CFG_ADVERTISED_LISTENERS="INTERNAL://${MY_POD_NAME}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $interBrokerPort }},CLIENT://${MY_POD_NAME}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $clientPort }},EXTERNAL://${EXTERNAL_ACCESS_IP}:${EXTERNAL_ACCESS_PORT}" + {{- end }} + {{- end }} + + {{- if (include "kafka.tlsEncryption" .) }} + if [[ -f "/certs/kafka.truststore.jks" ]] && [[ -f "/certs/kafka-${ID}.keystore.jks" ]]; then + mkdir -p /opt/bitnami/kafka/config/certs + cp "/certs/kafka.truststore.jks" "/opt/bitnami/kafka/config/certs/kafka.truststore.jks" + cp "/certs/kafka-${ID}.keystore.jks" "/opt/bitnami/kafka/config/certs/kafka.keystore.jks" + else + echo "Couldn't find the expected Java Key Stores (JKS) files! They are mandatory when encryption via TLS is enabled." + exit 1 + fi + {{- end }} + + exec /entrypoint.sh /run.sh diff --git a/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml b/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml new file mode 100755 index 000000000..790790b3f --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kafka.serviceAccountName" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml b/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml new file mode 100755 index 000000000..250bb5306 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.metrics.jmx.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kafka.fullname" . }}-jmx-metrics + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: kafka + endpoints: + - port: http-metrics + path: "/" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml b/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml new file mode 100755 index 000000000..951bf7c41 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.metrics.kafka.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kafka.fullname" . }}-metrics + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: metrics + endpoints: + - port: http-metrics + path: "/metrics" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml b/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml new file mode 100755 index 000000000..e9b5ce8f9 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml @@ -0,0 +1,435 @@ +{{- $replicaCount := int .Values.replicaCount }} +{{- $fullname := include "kafka.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} +{{- $interBrokerPort := .Values.service.internalPort }} +{{- $clientPort := .Values.service.port }} +{{- $interBrokerProtocol := include "kafka.listenerType" ( dict "protocol" .Values.auth.interBrokerProtocol ) -}} +{{- $clientProtocol := include "kafka.listenerType" ( dict "protocol" .Values.auth.clientProtocol ) -}} +{{- $loadBalancerIPListLength := len .Values.externalAccess.service.loadBalancerIPs }} +{{- if not (and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $loadBalancerIPListLength )) (eq .Values.externalAccess.service.type "LoadBalancer")) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podManagementPolicy: Parallel + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: {{- include "kafka.matchLabels" . | nindent 6 }} + app.kubernetes.io/component: kafka + serviceName: {{ template "kafka.fullname" . }}-headless + updateStrategy: + type: {{ .Values.updateStrategy | quote }} + {{- if (eq "OnDelete" .Values.updateStrategy) }} + rollingUpdate: null + {{- else if .Values.rollingUpdatePartition }} + rollingUpdate: + partition: {{ .Values.rollingUpdatePartition }} + {{- end }} + template: + metadata: + labels: {{- include "kafka.labels" . | nindent 8 }} + app.kubernetes.io/component: kafka + {{- if .Values.podLabels }} + {{- include "kafka.tplValue" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if or (include "kafka.createConfigmap" .) (include "kafka.createJaasSecret" .) .Values.externalAccess.enabled (include "kafka.metrics.jmx.createConfigmap" .) .Values.podAnnotations }} + annotations: + {{- if (include "kafka.createConfigmap" .) }} + checksum/configuration: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if (include "kafka.createJaasSecret" .) }} + checksum/secret: {{ include (print $.Template.BasePath "/jaas-secret.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.externalAccess.enabled }} + checksum/scripts: {{ include (print $.Template.BasePath "/scripts-configmap.yaml") . | sha256sum }} + {{- end }} + {{- if (include "kafka.metrics.jmx.createConfigmap" .) }} + checksum/jmx-configuration: {{ include (print $.Template.BasePath "/jmx-configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- include "kafka.tplValue" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + {{- end }} + spec: +{{- include "kafka.imagePullSecrets" . | indent 6 }} + {{- if .Values.affinity }} + affinity: {{- include "kafka.tplValue" ( dict "value" .Values.affinity "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "kafka.tplValue" ( dict "value" .Values.nodeSelector "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "kafka.tplValue" ( dict "value" .Values.tolerations "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- if .Values.podSecurityContext }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ template "kafka.serviceAccountName" . }} + {{- end }} + {{- if or (and .Values.volumePermissions.enabled .Values.persistence.enabled) (and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled) }} + initContainers: + {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }} + - name: volume-permissions + image: {{ include "kafka.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + args: + - -ec + - | + mkdir -p /bitnami/kafka + chown -R "{{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }}" "/bitnami/kafka" + securityContext: + runAsUser: 0 + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/kafka + {{- end }} + {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }} + - name: auto-discovery + image: {{ include "kafka.externalAccess.autoDiscovery.image" . }} + imagePullPolicy: {{ .Values.externalAccess.autoDiscovery.image.pullPolicy | quote }} + command: + - /scripts/auto-discovery.sh + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: SHARED_FILE + value: "/shared/info.txt" + {{- if .Values.externalAccess.autoDiscovery.resources }} + resources: {{- toYaml .Values.externalAccess.autoDiscovery.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: shared + mountPath: /shared + - name: scripts + mountPath: /scripts/auto-discovery.sh + subPath: auto-discovery.sh + {{- end }} + {{- end }} + containers: + - name: kafka + image: {{ include "kafka.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + command: {{- include "kafka.tplValue" (dict "value" .Values.command "context" $) | nindent 12 }} + {{- if .Values.args }} + args: {{- include "kafka.tplValue" (dict "value" .Values.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" .Values.image.debug | quote }} + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KAFKA_CFG_ZOOKEEPER_CONNECT + {{- if .Values.zookeeper.enabled }} + value: {{ include "kafka.zookeeper.fullname" . | quote }} + {{- else }} + value: {{ join "," .Values.externalZookeeper.servers | quote }} + {{- end }} + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: {{ .Values.interBrokerListenerName | quote }} + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + {{- if .Values.listenerSecurityProtocolMap }} + value: {{ .Values.listenerSecurityProtocolMap | quote }} + {{- else if .Values.externalAccess.enabled }} + value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }},EXTERNAL:{{ $clientProtocol }}" + {{- else }} + value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }}" + {{- end }} + {{- if or ($clientProtocol | regexFind "SASL") ($interBrokerProtocol | regexFind "SASL") .Values.auth.jaas.zookeeperUser }} + - name: KAFKA_CFG_SASL_ENABLED_MECHANISMS + value: {{ include "kafka.auth.saslMechanisms" ( dict "type" .Values.auth.saslMechanisms ) }} + - name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL + value: {{ upper .Values.auth.saslInterBrokerMechanism | quote }} + {{- end }} + - name: KAFKA_CFG_LISTENERS + {{- if .Values.listeners }} + value: {{ .Values.listeners }} + {{- else if .Values.externalAccess.enabled }} + value: "INTERNAL://:{{ $interBrokerPort }},CLIENT://:9092,EXTERNAL://:9094" + {{- else }} + value: "INTERNAL://:{{ $interBrokerPort }},CLIENT://:9092" + {{- end }} + {{- if .Values.externalAccess.enabled }} + {{- if .Values.externalAccess.autoDiscovery.enabled }} + - name: SHARED_FILE + value: "/shared/info.txt" + {{- end }} + {{- else }} + - name: KAFKA_CFG_ADVERTISED_LISTENERS + {{- if .Values.advertisedListeners }} + value: {{ .Values.advertisedListeners }} + {{- else }} + value: "INTERNAL://$(MY_POD_NAME).{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $interBrokerPort }},CLIENT://$(MY_POD_NAME).{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $clientPort }}" + {{- end }} + {{- end }} + - name: ALLOW_PLAINTEXT_LISTENER + value: {{ ternary "yes" "no" (or .Values.auth.enabled .Values.allowPlaintextListener) | quote }} + {{- if or (include "kafka.client.saslAuthentication" .) (include "kafka.interBroker.saslAuthentication" .) }} + - name: KAFKA_OPTS + value: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf" + {{- if (include "kafka.client.saslAuthentication" .) }} + - name: KAFKA_CLIENT_USERS + value: {{ join "," .Values.auth.jaas.clientUsers | quote }} + - name: KAFKA_CLIENT_PASSWORDS + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: client-passwords + {{- end }} + {{- if .Values.auth.jaas.zookeeperUser }} + - name: KAFKA_ZOOKEEPER_PROTOCOL + value: "SASL" + - name: KAFKA_ZOOKEEPER_USER + value: {{ .Values.auth.jaas.zookeeperUser | quote }} + - name: KAFKA_ZOOKEEPER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: zookeeper-password + {{- end }} + {{- if (include "kafka.interBroker.saslAuthentication" .) }} + - name: KAFKA_INTER_BROKER_USER + value: {{ .Values.auth.jaas.interBrokerUser | quote }} + - name: KAFKA_INTER_BROKER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "kafka.jaasSecretName" . }} + key: inter-broker-password + {{- end }} + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + - name: KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM + value: {{ .Values.auth.tlsEndpointIdentificationAlgorithm | quote }} + {{- if .Values.auth.jksPassword }} + - name: KAFKA_CERTIFICATE_PASSWORD + value: {{ .Values.auth.jksPassword | quote }} + {{- end }} + {{- end }} + {{- if .Values.metrics.jmx.enabled }} + - name: JMX_PORT + value: "5555" + {{- end }} + - name: KAFKA_CFG_DELETE_TOPIC_ENABLE + value: {{ .Values.deleteTopicEnable | quote }} + - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE + value: {{ .Values.autoCreateTopicsEnable | quote }} + - name: KAFKA_HEAP_OPTS + value: {{ .Values.heapOpts | quote }} + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES + value: {{ .Values.logFlushIntervalMessages | quote }} + - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS + value: {{ .Values.logFlushIntervalMs | quote }} + - name: KAFKA_CFG_LOG_RETENTION_BYTES + value: {{ .Values.logRetentionBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS + value: {{ .Values.logRetentionCheckIntervalMs | quote }} + - name: KAFKA_CFG_LOG_RETENTION_HOURS + value: {{ .Values.logRetentionHours | quote }} + - name: KAFKA_CFG_MESSAGE_MAX_BYTES + value: {{ .Values.maxMessageBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_LOG_SEGMENT_BYTES + value: {{ .Values.logSegmentBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_LOG_DIRS + value: {{ .Values.logsDirs | quote }} + - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR + value: {{ .Values.defaultReplicationFactor | quote }} + - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR + value: {{ .Values.offsetsTopicReplicationFactor | quote }} + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR + value: {{ .Values.transactionStateLogReplicationFactor | quote }} + - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR + value: {{ .Values.transactionStateLogMinIsr | quote }} + - name: KAFKA_CFG_NUM_IO_THREADS + value: {{ .Values.numIoThreads | quote }} + - name: KAFKA_CFG_NUM_NETWORK_THREADS + value: {{ .Values.numNetworkThreads | quote }} + - name: KAFKA_CFG_NUM_PARTITIONS + value: {{ .Values.numPartitions | quote }} + - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR + value: {{ .Values.numRecoveryThreadsPerDataDir | quote }} + - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES + value: {{ .Values.socketReceiveBufferBytes | quote }} + - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES + value: {{ .Values.socketRequestMaxBytes | replace "_" "" | quote }} + - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES + value: {{ .Values.socketSendBufferBytes | quote }} + - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS + value: {{ .Values.zookeeperConnectionTimeoutMs | quote }} + {{- if .Values.extraEnvVars }} + {{ include "kafka.tplValue" ( dict "value" .Values.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + ports: + - name: kafka-client + containerPort: 9092 + - name: kafka-internal + containerPort: {{ $interBrokerPort }} + {{- if .Values.externalAccess.enabled }} + - name: kafka-external + containerPort: 9094 + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + {{- else if .Values.customLivenessProbe }} + livenessProbe: {{- include "kafka.tplValue" (dict "value" .Values.customlivenessProbe "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + {{- else if .Values.customReadinessProbe }} + readinessProbe: {{- include "kafka.tplValue" (dict "value" .Values.customreadinessProbe "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/kafka + {{- if or .Values.config .Values.existingConfigmap }} + - name: kafka-config + mountPath: /bitnami/kafka/config/server.properties + subPath: server.properties + {{- end }} + {{- if or .Values.log4j .Values.existingLog4jConfigMap }} + - name: log4j-config + mountPath: /bitnami/kafka/config/log4j.properties + subPath: log4j.properties + {{- end }} + - name: scripts + mountPath: /scripts/setup.sh + subPath: setup.sh + {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }} + - name: shared + mountPath: /shared + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + - name: kafka-certificates + mountPath: /certs + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.metrics.jmx.enabled }} + - name: jmx-exporter + image: {{ template "kafka.metrics.jmx.image" . }} + imagePullPolicy: {{ .Values.metrics.jmx.image.pullPolicy | quote }} + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-kafka/jmx-kafka-prometheus.yml + ports: + - name: metrics + containerPort: 5556 + {{- if .Values.metrics.jmx.resources }} + resources: {{- toYaml .Values.metrics.jmx.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-kafka + {{- end }} + {{- if .Values.sidecars }} + {{- include "kafka.tplValue" (dict "value" .Values.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + {{- if or .Values.config .Values.existingConfigmap }} + - name: kafka-config + configMap: + name: {{ include "kafka.configmapName" . }} + {{- end }} + {{- if or .Values.log4j .Values.existingLog4jConfigMap }} + - name: log4j-config + configMap: + name: {{ include "kafka.log4j.configMapName" . }} + {{ end }} + - name: scripts + configMap: + name: {{ include "kafka.fullname" . }}-scripts + defaultMode: 0755 + {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }} + - name: shared + emptyDir: {} + {{- end }} + {{- if .Values.metrics.jmx.enabled }} + - name: jmx-config + configMap: + name: {{ include "kafka.metrics.jmx.configmapName" . }} + {{- end }} + {{- if (include "kafka.tlsEncryption" .) }} + - name: kafka-certificates + secret: + secretName: {{ include "kafka.jksSecretName" . }} + defaultMode: 256 + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- if not .Values.persistence.enabled }} + - name: data + emptyDir: {} +{{- else if .Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ printf "%s" (tpl .Values.persistence.existingClaim .) }} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + {{- if .Values.persistence.annotations }} + annotations: {{- include "kafka.tplValue" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{ include "kafka.storageClass" . | nindent 8 }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml b/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml new file mode 100755 index 000000000..eefe0046d --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml @@ -0,0 +1,52 @@ +{{- if .Values.externalAccess.enabled }} +{{- $fullName := include "kafka.fullname" . }} +{{- $replicaCount := .Values.replicaCount | int }} +{{- $root := . }} + +{{- range $i, $e := until $replicaCount }} +{{- $targetPod := printf "%s-%d" (printf "%s" $fullName) $i }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" $ }}-{{ $i }}-external + labels: {{- include "kafka.labels" $ | nindent 4 }} + app.kubernetes.io/component: kafka + pod: {{ $targetPod }} + {{- if $root.Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" $root.Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or $root.Values.externalAccess.service.annotations $root.Values.commonAnnotations }} + annotations: + {{- if $root.Values.externalAccess.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" $root.Values.externalAccess.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if $root.Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" $root.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ $root.Values.externalAccess.service.type }} + {{- if eq $root.Values.externalAccess.service.type "LoadBalancer" }} + {{- if not (empty $root.Values.externalAccess.service.loadBalancerIPs) }} + loadBalancerIP: {{ index $root.Values.externalAccess.service.loadBalancerIPs $i }} + {{- end }} + {{- if $root.Values.externalAccess.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml $root.Values.externalAccess.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: tcp-kafka + port: {{ $root.Values.externalAccess.service.port }} + {{- if not (empty $root.Values.externalAccess.service.nodePorts) }} + nodePort: {{ index $root.Values.externalAccess.service.nodePorts $i }} + {{- else }} + nodePort: null + {{- end }} + targetPort: kafka-external + selector: {{- include "kafka.matchLabels" $ | nindent 4 }} + app.kubernetes.io/component: kafka + statefulset.kubernetes.io/pod-name: {{ $targetPod }} +--- +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml b/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml new file mode 100755 index 000000000..e7c2e5e6e --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }}-headless + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-client + port: {{ .Values.service.port }} + protocol: TCP + targetPort: kafka-client + - name: tcp-internal + port: {{ .Values.service.internalPort }} + protocol: TCP + targetPort: kafka-internal + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: kafka diff --git a/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml b/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml new file mode 100755 index 000000000..189cb9ffd --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kafka.fullname" . }} + labels: {{- include "kafka.labels" . | nindent 4 }} + app.kubernetes.io/component: kafka + {{- if .Values.commonLabels }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if or .Values.service.annotations .Values.commonAnnotations }} + annotations: + {{- if .Values.service.annotations }} + {{ include "kafka.tplValue" ( dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + {{- include "kafka.tplValue" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if eq .Values.service.type "LoadBalancer" }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: tcp-client + port: {{ .Values.service.port }} + protocol: TCP + targetPort: kafka-client + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.client)) }} + nodePort: {{ .Values.service.nodePorts.client }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if and .Values.externalAccess.enabled (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) }} + - name: tcp-external + port: {{ .Values.service.externalPort }} + protocol: TCP + targetPort: kafka-external + {{- if (not (empty .Values.service.nodePorts.external)) }} + nodePort: {{ .Values.service.nodePorts.external }} + {{- end }} + {{- end }} + selector: {{- include "kafka.matchLabels" . | nindent 4 }} + app.kubernetes.io/component: kafka diff --git a/scripts/helm/helm/databases/charts/kafka/values-production.yaml b/scripts/helm/helm/databases/charts/kafka/values-production.yaml new file mode 100755 index 000000000..af6f43dba --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/values-production.yaml @@ -0,0 +1,931 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Kafka image version +## ref: https://hub.docker.com/r/bitnami/kafka/tags/ +## +image: + registry: docker.io + repository: bitnami/kafka + tag: 2.6.0-debian-10-r30 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Set to true if you would like to see extra information on logs + ## + debug: false + +## String to partially override kafka.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override kafka.fullname template +## +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Kafka Configuration +## Specify content for server.properties +## The server.properties is auto-generated based on other parameters when this paremeter is not specified +## +## Example: +## config: |- +## broker.id=-1 +## listeners=PLAINTEXT://:9092 +## advertised.listeners=PLAINTEXT://KAFKA_IP:9092 +## num.network.threads=3 +## num.io.threads=8 +## socket.send.buffer.bytes=102400 +## socket.receive.buffer.bytes=102400 +## socket.request.max.bytes=104857600 +## log.dirs=/bitnami/kafka/data +## num.partitions=1 +## num.recovery.threads.per.data.dir=1 +## offsets.topic.replication.factor=1 +## transaction.state.log.replication.factor=1 +## transaction.state.log.min.isr=1 +## log.flush.interval.messages=10000 +## log.flush.interval.ms=1000 +## log.retention.hours=168 +## log.retention.bytes=1073741824 +## log.segment.bytes=1073741824 +## log.retention.check.interval.ms=300000 +## zookeeper.connect=ZOOKEEPER_SERVICE_NAME +## zookeeper.connection.timeout.ms=6000 +## group.initial.rebalance.delay.ms=0 +## +# config: + +## ConfigMap with Kafka Configuration +## NOTE: This will override config +## +# existingConfigmap: + +## Kafka Log4J Configuration +## An optional log4j.properties file to overwrite the default of the Kafka brokers. +## See an example log4j.properties at: +## https://github.com/apache/kafka/blob/trunk/config/log4j.properties +## +# log4j: + +## Kafka Log4j ConfigMap +## The name of an existing ConfigMap containing a log4j.properties file. +## NOTE: this will override log4j. +## +# existingLog4jConfigMap: + +## Kafka's Java Heap size +## +heapOpts: -Xmx1024m -Xms1024m + +## Switch to enable topic deletion or not. +## +deleteTopicEnable: false + +## Switch to enable auto creation of topics. +## Enabling auto creation of topics not recommended for production or similar environments. +## +autoCreateTopicsEnable: false + +## The number of messages to accept before forcing a flush of data to disk. +## +logFlushIntervalMessages: 10000 + +## The maximum amount of time a message can sit in a log before we force a flush. +## +logFlushIntervalMs: 1000 + +## A size-based retention policy for logs. +## +logRetentionBytes: _1073741824 + +## The interval at which log segments are checked to see if they can be deleted. +## +logRetentionCheckIntervalMs: 300000 + +## The minimum age of a log file to be eligible for deletion due to age. +## +logRetentionHours: 168 + +## The maximum size of a log segment file. When this size is reached a new log segment will be created. +## +logSegmentBytes: _1073741824 + +## A comma separated list of directories under which to store log files. +## +logsDirs: /bitnami/kafka/data + +## The largest record batch size allowed by Kafka +## +maxMessageBytes: _1000012 + +## Default replication factors for automatically created topics +## +defaultReplicationFactor: 3 + +## The replication factor for the offsets topic +## +offsetsTopicReplicationFactor: 3 + +## The replication factor for the transaction topic +## +transactionStateLogReplicationFactor: 3 + +## Overridden min.insync.replicas config for the transaction topic +## +transactionStateLogMinIsr: 3 + +## The number of threads doing disk I/O. +## +numIoThreads: 8 + +## The number of threads handling network requests. +## +numNetworkThreads: 3 + +## The default number of log partitions per topic. +## +numPartitions: 1 + +## The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +## +numRecoveryThreadsPerDataDir: 1 + +## The receive buffer (SO_RCVBUF) used by the socket server. +## +socketReceiveBufferBytes: 102400 + +## The maximum size of a request that the socket server will accept (protection against OOM). +## +socketRequestMaxBytes: _104857600 + +## The send buffer (SO_SNDBUF) used by the socket server. +## +socketSendBufferBytes: 102400 + +## Timeout in ms for connecting to zookeeper. +## +zookeeperConnectionTimeoutMs: 6000 + +## Command and args for running the container. Use array form +## +command: + - /scripts/setup.sh +args: + +## All the parameters from the configuration file can be overwritten by using environment variables with this format: KAFKA_CFG_{KEY} +## ref: https://github.com/bitnami/bitnami-docker-kafka#configuration +## Example: +## extraEnvVars: +## - name: KAFKA_CFG_BACKGROUND_THREADS +## value: "10" +## +extraEnvVars: [] + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Examples: +# extraVolumes: +# - name: kafka-jaas +# secret: +# secretName: kafka-jaas +# extraVolumeMounts: +# - name: kafka-jaas +# mountPath: /bitnami/kafka/config/kafka_jaas.conf +# subPath: kafka_jaas.conf +extraVolumes: [] +extraVolumeMounts: [] + +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: [] + +## Authentication parameteres +## https://github.com/bitnami/bitnami-docker-kafka#security +## +auth: + ## Authentication protocol for client and inter-broker communications + ## Supported values: 'plaintext', 'tls', 'mtls', 'sasl' and 'sasl_tls' + ## This table shows the security provided on each protocol: + ## | Method | Authentication | Encryption via TLS | + ## | plaintext | None | No | + ## | tls | None | Yes | + ## | mtls | Yes (two-way authentication) | Yes | + ## | sasl | Yes (via SASL) | No | + ## | sasl_tls | Yes (via SASL) | Yes | + ## + clientProtocol: sasl + interBrokerProtocol: sasl + + ## Allowed SASL mechanisms when clientProtocol or interBrokerProtocol are using either sasl or sasl_tls + ## + saslMechanisms: plain,scram-sha-256,scram-sha-512 + ## SASL mechanism for inter broker communication + ## + saslInterBrokerMechanism: plain + + ## Name of the existing secret containing the truststore and + ## one keystore per Kafka broker you have in the Kafka cluster. + ## MANDATORY when 'tls', 'mtls', or 'sasl_tls' authentication protocols are used. + ## Create this secret following the steps below: + ## 1) Generate your trustore and keystore files. Helpful script: https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh + ## 2) Rename your truststore to `kafka.truststore.jks`. + ## 3) Rename your keystores to `kafka-X.keystore.jks` where X is the ID of each Kafka broker. + ## 4) Run the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-file=./kafka.truststore.jks --from-file=./kafka-0.keystore.jks --from-file=./kafka-1.keystore.jks ... + ## Alternatively, you can put your JKS files under the files/jks directory + ## + # jksSecret: + + ## Password to access the JKS files when they are password-protected. + ## + # jksPassword: + + ## The endpoint identification algorithm used by clients to validate server host name. + ## Disable server host name verification by setting it to an empty string + ## See: https://docs.confluent.io/current/kafka/authentication_ssl.html#optional-settings + ## + tlsEndpointIdentificationAlgorithm: https + + ## JAAS configuration for SASL authentication + ## MANDATORY when method is 'sasl', or 'sasl_tls' + ## + jaas: + ## Kafka client user list + ## + ## clientUsers: + ## - user1 + ## - user2 + ## + clientUsers: + - user + + ## Kafka client passwords + ## + ## clientPasswords: + ## - password1 + ## - password2 + ## + clientPasswords: [] + + ## Kafka inter broker communication user + ## + interBrokerUser: admin + + ## Kafka inter broker communication password + ## + interBrokerPassword: "" + + ## Kafka Zookeeper user + ## + zookeeperUser: zookeeperUser + + ## Kafka Zookeeper password + ## + zookeeperPassword: zookeeperPassword + + ## Name of the existing secret containing credentials for clientUsers, interBrokerUser and zookeeperUser. + ## Create this secret running the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-literal=client-password=CLIENT_PASSWORD1,CLIENT_PASSWORD2 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=zookeeper-password=ZOOKEEPER_PASSWORD + ## + # existingSecret: + +## The address(es) the socket server listens on. +## When it's set to an empty array, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +listeners: [] + +## The address(es) (hostname:port) the brokers will advertise to producers and consumers. +## When it's set to an empty array, the advertised listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +advertisedListeners: [] + +## The listener->protocol mapping +## When it's nil, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +# listenerSecurityProtocolMap: + +## Allow to use the PLAINTEXT listener. +## +allowPlaintextListener: false + +## Name of listener used for communication between brokers. +## +interBrokerListenerName: INTERNAL + +## Number of Kafka brokers to deploy +## +replicaCount: 3 + +## StrategyType, can be set to RollingUpdate or OnDelete by default. +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## Pod labels. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +## Pod annotations. Evaluated as a template +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} + +## Name of the priority class to be used by kafka pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: "" + +## Affinity for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Kafka pods' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## +podSecurityContext: + fsGroup: 1001 + runAsUser: 1001 + +## Kafka containers' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## Example: +## containerSecurityContext: +## capabilities: +## drop: ["NET_RAW"] +## readOnlyRootFilesystem: true +## +containerSecurityContext: {} + +## Kafka containers' resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 250m + # memory: 1Gi + requests: {} + # cpu: 250m + # memory: 256Mi + +## Kafka containers' liveness and readiness probes. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes +## +livenessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 10 + timeoutSeconds: 5 + # failureThreshold: 3 + # periodSeconds: 10 + # successThreshold: 1 +readinessProbe: + tcpSocket: + port: kafka-client + initialDelaySeconds: 5 + failureThreshold: 6 + timeoutSeconds: 5 + # periodSeconds: 10 + # successThreshold: 1 + +## Pod Disruption Budget configuration +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +pdb: + create: true + ## Min number of pods that must still be available after the eviction + ## + # minAvailable: 1 + ## Max number of pods that can be unavailable after the eviction + ## + maxUnavailable: 1 + +## Add sidecars to the pod. +## Example: +## sidecars: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +sidecars: {} + +## Service parameters +## +service: + ## Service type + ## + type: ClusterIP + ## Kafka port for client connections + ## + port: 9092 + ## Kafka port for inter-broker connections + ## + internalPort: 9093 + ## Kafka port for external connections + ## + externalPort: 9094 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + client: "" + external: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## External Access to Kafka brokers configuration +## +externalAccess: + ## Enable Kubernetes external cluster access to Kafka brokers + ## + enabled: false + + ## External IPs auto-discovery configuration + ## An init container is used to auto-detect LB IPs or node ports by querying the K8s API + ## Note: RBAC might be required + ## + autoDiscovery: + ## Enable external IP/ports auto-discovery + ## + enabled: false + ## Bitnami Kubectl image + ## ref: https://hub.docker.com/r/bitnami/kubectl/tags/ + ## + image: + registry: docker.io + repository: bitnami/kubectl + tag: 1.17.12-debian-10-r3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Parameters to configure K8s service(s) used to externally access Kafka brokers + ## A new service per broker will be created + ## + service: + ## Service type. Allowed values: LoadBalancer or NodePort + ## + type: LoadBalancer + ## Port used when service type is LoadBalancer + ## + port: 9094 + ## Array of load balancer IPs for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## loadBalancerIPs: + ## - X.X.X.X + ## - Y.Y.Y.Y + ## + loadBalancerIPs: [] + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Array of node ports used for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## nodePorts: + ## - 30001 + ## - 30002 + ## + nodePorts: [] + ## When service type is NodePort, you can specify the domain used for Kafka advertised listeners. + ## If not specified, the container will try to get the kubernetes node external IP + ## + # domain: mydomain.com + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## Persistence paramaters +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + ## PV Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. + ## + # storageClass: "-" + ## PV Access Mode + ## + accessModes: + - ReadWriteOnce + ## PVC size + ## + size: 8Gi + ## PVC annotations + ## + annotations: {} + +## Init Container paramaters +## Change the owner and group of the persistent volume(s) mountpoint(s) to 'runAsUser:fsGroup' on each component +## values from the securityContext section of the component +## +volumePermissions: + enabled: false + ## Bitnami Minideb image + ## ref: https://hub.docker.com/r/bitnami/minideb/tags/ + ## + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## Kafka pods ServiceAccount +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: true + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fluentd.fullname template + ## + # name: + +## Role Based Access +## ref: https://kubernetes.io/docs/admin/authorization/rbac/ +## +rbac: + ## Specifies whether RBAC rules should be created + ## binding Kafka ServiceAccount to a role + ## that allows Kafka pods querying the K8s API + ## + create: false + +## Prometheus Exporters / Metrics +## +metrics: + ## Prometheus Kafka Exporter: exposes complimentary metrics to JMX Exporter + ## + kafka: + enabled: true + + ## Bitnami Kafka exporter image + ## ref: https://hub.docker.com/r/bitnami/kafka-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/kafka-exporter + tag: 1.2.0-debian-10-r220 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Extra flags to be passed to Kafka exporter + ## Example: + ## extraFlags: + ## tls.insecure-skip-tls-verify: "" + ## web.telemetry-path: "/metrics" + ## + extraFlags: {} + + ## Name of the existing secret containing the optional certificate and key files + ## for Kafka Exporter client authentication + ## + # certificatesSecret: + + ## Prometheus Kafka Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## Kafka Exporter Service type + ## + type: ClusterIP + ## Kafka Exporter Prometheus port + ## + port: 9308 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the Kafka Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.kafka.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus JMX Exporter: exposes the majority of Kafkas metrics + ## + jmx: + enabled: true + + ## Bitnami JMX exporter image + ## ref: https://hub.docker.com/r/bitnami/jmx-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/jmx-exporter + tag: 0.14.0-debian-10-r15 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Prometheus JMX Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## JMX Exporter Service type + ## + type: ClusterIP + ## JMX Exporter Prometheus port + ## + port: 5556 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the JMX Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.jmx.service.port }}" + prometheus.io/path: "/" + + ## JMX Whitelist Objects, can be set to control which JMX metrics are exposed. Only whitelisted + ## values will be exposed via JMX Exporter. They must also be exposed via Rules. To expose all metrics + ## (warning its crazy excessive and they aren't formatted in a prometheus style) (1) `whitelistObjectNames: []` + ## (2) commented out above `overrideConfig`. + ## + whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* + + ## Prometheus JMX exporter configuration + ## Specify content for jmx-kafka-prometheus.yml. Evaluated as a template + ## + ## Credits to the incubator/kafka chart for the JMX configuration. + ## https://github.com/helm/charts/tree/master/incubator/kafka + ## + config: |- + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false + {{- if .Values.metrics.jmx.whitelistObjectNames }} + whitelistObjectNames: ["{{ join "\",\"" .Values.metrics.jmx.whitelistObjectNames }}"] + {{- end }} + + ## ConfigMap with Prometheus JMX exporter configuration + ## NOTE: This will override metrics.jmx.config + ## + # existingConfigmap: + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace in which Prometheus is running + ## + # namespace: monitoring + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + +## +## Zookeeper chart configuration +## +## https://github.com/bitnami/charts/blob/master/bitnami/zookeeper/values.yaml +## +zookeeper: + enabled: true + auth: + ## Enable Zookeeper auth + ## + enabled: true + ## User that will use Zookeeper clients to auth + ## + clientUser: zookeeperUser + ## Password that will use Zookeeper clients to auth + ## + clientPassword: zookeeperPassword + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + serverUsers: zookeeperUser + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + serverPasswords: zookeeperPassword + metrics: + enabled: true + +## This value is only used when zookeeper.enabled is set to false +## +externalZookeeper: + ## Server or list of external zookeeper servers to use. + ## + servers: [] diff --git a/scripts/helm/helm/databases/charts/kafka/values.yaml b/scripts/helm/helm/databases/charts/kafka/values.yaml new file mode 100755 index 000000000..154d71bd5 --- /dev/null +++ b/scripts/helm/helm/databases/charts/kafka/values.yaml @@ -0,0 +1,934 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +# global: +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami Kafka image version +## ref: https://hub.docker.com/r/bitnami/kafka/tags/ +## +image: + registry: docker.io + repository: bitnami/kafka + tag: 2.6.0-debian-10-r30 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Set to true if you would like to see extra information on logs + ## + debug: false + +## String to partially override kafka.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override kafka.fullname template +## +# fullnameOverride: + +## Kubernetes Cluster Domain +## +clusterDomain: cluster.local + +## Add labels to all the deployed resources +## +commonLabels: {} + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +## Kafka Configuration +## Specify content for server.properties +## The server.properties is auto-generated based on other parameters when this paremeter is not specified +## +## Example: +## config: |- +## broker.id=-1 +## listeners=PLAINTEXT://:9092 +## advertised.listeners=PLAINTEXT://KAFKA_IP:9092 +## num.network.threads=3 +## num.io.threads=8 +## socket.send.buffer.bytes=102400 +## socket.receive.buffer.bytes=102400 +## socket.request.max.bytes=104857600 +## log.dirs=/bitnami/kafka/data +## num.partitions=1 +## num.recovery.threads.per.data.dir=1 +## offsets.topic.replication.factor=1 +## transaction.state.log.replication.factor=1 +## transaction.state.log.min.isr=1 +## log.flush.interval.messages=10000 +## log.flush.interval.ms=1000 +## log.retention.hours=168 +## log.retention.bytes=1073741824 +## log.segment.bytes=1073741824 +## log.retention.check.interval.ms=300000 +## zookeeper.connect=ZOOKEEPER_SERVICE_NAME +## zookeeper.connection.timeout.ms=6000 +## group.initial.rebalance.delay.ms=0 +## +# config: + +## ConfigMap with Kafka Configuration +## NOTE: This will override config +## +# existingConfigmap: + +## Kafka Log4J Configuration +## An optional log4j.properties file to overwrite the default of the Kafka brokers. +## See an example log4j.properties at: +## https://github.com/apache/kafka/blob/trunk/config/log4j.properties +## +# log4j: + +## Kafka Log4j ConfigMap +## The name of an existing ConfigMap containing a log4j.properties file. +## NOTE: this will override log4j. +## +# existingLog4jConfigMap: + +## Kafka's Java Heap size +## +heapOpts: -Xmx1024m -Xms1024m + +## Switch to enable topic deletion or not. +## +deleteTopicEnable: false + +## Switch to enable auto creation of topics. +## Enabling auto creation of topics not recommended for production or similar environments. +## +autoCreateTopicsEnable: true + +## The number of messages to accept before forcing a flush of data to disk. +## +logFlushIntervalMessages: 10000 + +## The maximum amount of time a message can sit in a log before we force a flush. +## +logFlushIntervalMs: 1000 + +## A size-based retention policy for logs. +## +logRetentionBytes: _1073741824 + +## The interval at which log segments are checked to see if they can be deleted. +## +logRetentionCheckIntervalMs: 300000 + +## The minimum age of a log file to be eligible for deletion due to age. +## +logRetentionHours: 168 + +## The maximum size of a log segment file. When this size is reached a new log segment will be created. +## +logSegmentBytes: _1073741824 + +## A comma separated list of directories under which to store log files. +## +logsDirs: /bitnami/kafka/data + +## The largest record batch size allowed by Kafka +## +maxMessageBytes: _1000012 + +## Default replication factors for automatically created topics +## +defaultReplicationFactor: 1 + +## The replication factor for the offsets topic +## +offsetsTopicReplicationFactor: 1 + +## The replication factor for the transaction topic +## +transactionStateLogReplicationFactor: 1 + +## Overridden min.insync.replicas config for the transaction topic +## +transactionStateLogMinIsr: 1 + +## The number of threads doing disk I/O. +## +numIoThreads: 8 + +## The number of threads handling network requests. +## +numNetworkThreads: 3 + +## The default number of log partitions per topic. +## +numPartitions: 1 + +## The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +## +numRecoveryThreadsPerDataDir: 1 + +## The receive buffer (SO_RCVBUF) used by the socket server. +## +socketReceiveBufferBytes: 102400 + +## The maximum size of a request that the socket server will accept (protection against OOM). +## +socketRequestMaxBytes: _104857600 + +## The send buffer (SO_SNDBUF) used by the socket server. +## +socketSendBufferBytes: 102400 + +## Timeout in ms for connecting to zookeeper. +## +zookeeperConnectionTimeoutMs: 6000 + +## Command and args for running the container. Use array form +## +command: + - /scripts/setup.sh +args: + +## All the parameters from the configuration file can be overwritten by using environment variables with this format: KAFKA_CFG_{KEY} +## ref: https://github.com/bitnami/bitnami-docker-kafka#configuration +## Example: +## extraEnvVars: +## - name: KAFKA_CFG_BACKGROUND_THREADS +## value: "10" +## +extraEnvVars: [] + +## extraVolumes and extraVolumeMounts allows you to mount other volumes +## Examples: +# extraVolumes: +# - name: kafka-jaas +# secret: +# secretName: kafka-jaas +# extraVolumeMounts: +# - name: kafka-jaas +# mountPath: /bitnami/kafka/config/kafka_jaas.conf +# subPath: kafka_jaas.conf +extraVolumes: [] +extraVolumeMounts: [] + +## Extra objects to deploy (value evaluated as a template) +## +extraDeploy: [] + +## Authentication parameteres +## https://github.com/bitnami/bitnami-docker-kafka#security +## +auth: + ## Authentication protocol for client and inter-broker communications + ## Supported values: 'plaintext', 'tls', 'mtls', 'sasl' and 'sasl_tls' + ## This table shows the security provided on each protocol: + ## | Method | Authentication | Encryption via TLS | + ## | plaintext | None | No | + ## | tls | None | Yes | + ## | mtls | Yes (two-way authentication) | Yes | + ## | sasl | Yes (via SASL) | No | + ## | sasl_tls | Yes (via SASL) | Yes | + ## + clientProtocol: plaintext + interBrokerProtocol: plaintext + + ## Allowed SASL mechanisms when clientProtocol or interBrokerProtocol are using either sasl or sasl_tls + ## + saslMechanisms: plain,scram-sha-256,scram-sha-512 + ## SASL mechanism for inter broker communication + ## + saslInterBrokerMechanism: plain + + ## Name of the existing secret containing the truststore and + ## one keystore per Kafka broker you have in the Kafka cluster. + ## MANDATORY when 'tls', 'mtls', or 'sasl_tls' authentication protocols are used. + ## Create this secret following the steps below: + ## 1) Generate your trustore and keystore files. Helpful script: https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh + ## 2) Rename your truststore to `kafka.truststore.jks`. + ## 3) Rename your keystores to `kafka-X.keystore.jks` where X is the ID of each Kafka broker. + ## 4) Run the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-file=./kafka.truststore.jks --from-file=./kafka-0.keystore.jks --from-file=./kafka-1.keystore.jks ... + ## Alternatively, you can put your JKS files under the files/jks directory + ## + # jksSecret: + + ## Password to access the JKS files when they are password-protected. + ## + # jksPassword: + + ## The endpoint identification algorithm used by clients to validate server host name. + ## Disable server host name verification by setting it to an empty string + ## See: https://docs.confluent.io/current/kafka/authentication_ssl.html#optional-settings + ## + tlsEndpointIdentificationAlgorithm: https + + ## JAAS configuration for SASL authentication + ## MANDATORY when method is 'sasl', or 'sasl_tls' + ## + jaas: + ## Kafka client user list + ## + ## clientUsers: + ## - user1 + ## - user2 + ## + clientUsers: + - user + + ## Kafka client passwords. This is mandatory if more than one user is specified in clientUsers. + ## + ## clientPasswords: + ## - password1 + ## - password2" + ## + clientPasswords: [] + + ## Kafka inter broker communication user + ## + interBrokerUser: admin + + ## Kafka inter broker communication password + ## + interBrokerPassword: "" + + ## Kafka Zookeeper user + ## + # zookeeperUser: + + ## Kafka Zookeeper password + ## + # zookeeperPassword: + + ## Name of the existing secret containing credentials for clientUsers, interBrokerUser and zookeeperUser. + ## Create this secret running the command below where SECRET_NAME is the name of the secret you want to create: + ## kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1,CLIENT_PASSWORD2 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=zookeeper-password=ZOOKEEPER_PASSWORD + ## + # existingSecret: + +## The address(es) the socket server listens on. +## When it's set to an empty array, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +listeners: [] + +## The address(es) (hostname:port) the brokers will advertise to producers and consumers. +## When it's set to an empty array, the advertised listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +advertisedListeners: [] + +## The listener->protocol mapping +## When it's nil, the listeners will be configured +## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters) +## +# listenerSecurityProtocolMap: + +## Allow to use the PLAINTEXT listener. +## +allowPlaintextListener: true + +## Name of listener used for communication between brokers. +## +interBrokerListenerName: INTERNAL + +## Number of Kafka brokers to deploy +## +replicaCount: 2 + +## StrategyType, can be set to RollingUpdate or OnDelete by default. +## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets +## +updateStrategy: RollingUpdate + +## Partition update strategy +## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions +## +# rollingUpdatePartition: + +## Pod labels. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +## +podLabels: {} + +## Pod annotations. Evaluated as a template +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} + +## Name of the priority class to be used by kafka pods, priority class needs to be created beforehand +## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +priorityClassName: "" + +## Affinity for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Kafka pods' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## +podSecurityContext: + fsGroup: 1001 + runAsUser: 1001 + +## Kafka containers' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## Example: +## containerSecurityContext: +## capabilities: +## drop: ["NET_RAW"] +## readOnlyRootFilesystem: true +## +containerSecurityContext: {} + +## Kafka containers' resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 250m + # memory: 1Gi + requests: {} + # cpu: 250m + # memory: 256Mi + +## Kafka containers' liveness and readiness probes. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes +## +livenessProbe: + enabled: true + initialDelaySeconds: 10 + timeoutSeconds: 5 + # failureThreshold: 3 + # periodSeconds: 10 + # successThreshold: 1 +readinessProbe: + enabled: true + initialDelaySeconds: 5 + failureThreshold: 6 + timeoutSeconds: 5 + # periodSeconds: 10 + # successThreshold: 1 + +## Custom liveness/readiness probes that will override the default ones +## +customLivenessProbe: {} +customReadinessProbe: {} + +## Pod Disruption Budget configuration +## The PDB will only be created if replicaCount is greater than 1 +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions +## +pdb: + create: true + ## Min number of pods that must still be available after the eviction + ## + # minAvailable: 1 + ## Max number of pods that can be unavailable after the eviction + ## + maxUnavailable: 1 + +## Add sidecars to the pod. +## Example: +## sidecars: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +sidecars: {} + +## Service parameters +## +service: + ## Service type + ## + type: ClusterIP + ## Kafka port for client connections + ## + port: 9092 + ## Kafka port for inter-broker connections + ## + internalPort: 9093 + ## Kafka port for external connections + ## + externalPort: 9094 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + client: "" + external: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## External Access to Kafka brokers configuration +## +externalAccess: + ## Enable Kubernetes external cluster access to Kafka brokers + ## + enabled: false + + ## External IPs auto-discovery configuration + ## An init container is used to auto-detect LB IPs or node ports by querying the K8s API + ## Note: RBAC might be required + ## + autoDiscovery: + ## Enable external IP/ports auto-discovery + ## + enabled: false + ## Bitnami Kubectl image + ## ref: https://hub.docker.com/r/bitnami/kubectl/tags/ + ## + image: + registry: docker.io + repository: bitnami/kubectl + tag: 1.17.12-debian-10-r3 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Parameters to configure K8s service(s) used to externally access Kafka brokers + ## A new service per broker will be created + ## + service: + ## Service type. Allowed values: LoadBalancer or NodePort + ## + type: LoadBalancer + ## Port used when service type is LoadBalancer + ## + port: 9094 + ## Array of load balancer IPs for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## loadBalancerIPs: + ## - X.X.X.X + ## - Y.Y.Y.Y + ## + loadBalancerIPs: [] + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Array of node ports used for each Kafka broker. Length must be the same as replicaCount + ## Example: + ## nodePorts: + ## - 30001 + ## - 30002 + ## + nodePorts: [] + ## When service type is NodePort, you can specify the domain used for Kafka advertised listeners. + ## If not specified, the container will try to get the kubernetes node external IP + ## + # domain: mydomain.com + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +## Persistence paramaters +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template + ## + # existingClaim: + ## PV Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. + ## + # storageClass: "-" + ## PV Access Mode + ## + accessModes: + - ReadWriteOnce + ## PVC size + ## + size: 8Gi + ## PVC annotations + ## + annotations: {} + +## Init Container paramaters +## Change the owner and group of the persistent volume(s) mountpoint(s) to 'runAsUser:fsGroup' on each component +## values from the securityContext section of the component +## +volumePermissions: + enabled: false + ## Bitnami Minideb image + ## ref: https://hub.docker.com/r/bitnami/minideb/tags/ + ## + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init Container resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## Kafka pods ServiceAccount +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: true + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fluentd.fullname template + ## + # name: + +## Role Based Access +## ref: https://kubernetes.io/docs/admin/authorization/rbac/ +## +rbac: + ## Specifies whether RBAC rules should be created + ## binding Kafka ServiceAccount to a role + ## that allows Kafka pods querying the K8s API + ## + create: false + +## Prometheus Exporters / Metrics +## +metrics: + ## Prometheus Kafka Exporter: exposes complimentary metrics to JMX Exporter + ## + kafka: + enabled: false + + ## Bitnami Kafka exporter image + ## ref: https://hub.docker.com/r/bitnami/kafka-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/kafka-exporter + tag: 1.2.0-debian-10-r220 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Extra flags to be passed to Kafka exporter + ## Example: + ## extraFlags: + ## tls.insecure-skip-tls-verify: "" + ## web.telemetry-path: "/metrics" + ## + extraFlags: {} + + ## Name of the existing secret containing the optional certificate and key files + ## for Kafka Exporter client authentication + ## + # certificatesSecret: + + ## Prometheus Kafka Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## Kafka Exporter Service type + ## + type: ClusterIP + ## Kafka Exporter Prometheus port + ## + port: 9308 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the Kafka Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.kafka.service.port }}" + prometheus.io/path: "/metrics" + + ## Prometheus JMX Exporter: exposes the majority of Kafkas metrics + ## + jmx: + enabled: false + + ## Bitnami JMX exporter image + ## ref: https://hub.docker.com/r/bitnami/jmx-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/jmx-exporter + tag: 0.14.0-debian-10-r15 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + + ## Prometheus JMX Exporter' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + + ## Service configuration + ## + service: + ## JMX Exporter Service type + ## + type: ClusterIP + ## JMX Exporter Prometheus port + ## + port: 5556 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePort: "" + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the Cluster IP to use + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address + ## + # clusterIP: None + ## Annotations for the JMX Exporter Prometheus metrics service + ## + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.metrics.jmx.service.port }}" + prometheus.io/path: "/" + + ## JMX Whitelist Objects, can be set to control which JMX metrics are exposed. Only whitelisted + ## values will be exposed via JMX Exporter. They must also be exposed via Rules. To expose all metrics + ## (warning its crazy excessive and they aren't formatted in a prometheus style) (1) `whitelistObjectNames: []` + ## (2) commented out above `overrideConfig`. + ## + whitelistObjectNames: + - kafka.controller:* + - kafka.server:* + - java.lang:* + - kafka.network:* + - kafka.log:* + + ## Prometheus JMX exporter configuration + ## Specify content for jmx-kafka-prometheus.yml. Evaluated as a template + ## + ## Credits to the incubator/kafka chart for the JMX configuration. + ## https://github.com/helm/charts/tree/master/incubator/kafka + ## + config: |- + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false + {{- if .Values.metrics.jmx.whitelistObjectNames }} + whitelistObjectNames: ["{{ join "\",\"" .Values.metrics.jmx.whitelistObjectNames }}"] + {{- end }} + + ## ConfigMap with Prometheus JMX exporter configuration + ## NOTE: This will override metrics.jmx.config + ## + # existingConfigmap: + + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + enabled: false + ## Namespace in which Prometheus is running + ## + # namespace: monitoring + + ## Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # interval: 10s + + ## Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## + # scrapeTimeout: 10s + + ## ServiceMonitor selector labels + ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration + ## + # selector: + # prometheus: my-prometheus + +## +## Zookeeper chart configuration +## +## https://github.com/bitnami/charts/blob/master/bitnami/zookeeper/values.yaml +## +zookeeper: + enabled: true + auth: + ## Enable Zookeeper auth + ## + enabled: false + ## User that will use Zookeeper clients to auth + ## + # clientUser: + ## Password that will use Zookeeper clients to auth + ## + # clientPassword: + ## Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin" + ## + # serverUsers: + ## Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin" + ## + # serverPasswords: + +## This value is only used when zookeeper.enabled is set to false +## +externalZookeeper: + ## Server or list of external zookeeper servers to use. + ## + servers: [] diff --git a/scripts/helm/helm/databases/charts/minio/.helmignore b/scripts/helm/helm/databases/charts/minio/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/scripts/helm/helm/databases/charts/minio/Chart.yaml b/scripts/helm/helm/databases/charts/minio/Chart.yaml new file mode 100755 index 000000000..395e3ac91 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/Chart.yaml @@ -0,0 +1,24 @@ +annotations: + category: Infrastructure +apiVersion: v1 +appVersion: 2020.10.9 +description: MinIO is an object storage server, compatible with Amazon S3 cloud storage + service, mainly used for storing unstructured data (such as photos, videos, log + files, etc.) +engine: gotpl +home: https://github.com/bitnami/charts/tree/master/bitnami/minio +icon: https://bitnami.com/assets/stacks/minio/img/minio-stack-220x234.png +keywords: +- minio +- storage +- object-storage +- s3 +- cluster +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: minio +sources: +- https://github.com/bitnami/bitnami-docker-minio +- https://min.io +version: 3.7.14 diff --git a/scripts/helm/helm/databases/charts/minio/README.md b/scripts/helm/helm/databases/charts/minio/README.md new file mode 100755 index 000000000..4a4c45c65 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/README.md @@ -0,0 +1,259 @@ +# MinIO + +[MinIO](https://min.io) is an object storage server, compatible with Amazon S3 cloud storage service, mainly used for storing unstructured data (such as photos, videos, log files, etc.) + +## TL;DR + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/minio +``` + +## Introduction + +This chart bootstraps a [MinIO](https://github.com/bitnami/bitnami-docker-minio) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ +- PV provisioner support in the underlying infrastructure +- ReadWriteMany volumes for deployment scaling + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/minio +``` + +These commands deploy MinIO on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +The following table lists the configurable parameters of the MinIO chart and their default values. + +| Parameter | Description | Default | +|--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | +| `global.minio.existingSecret` | Name of existing secret to use for MinIO credentials (overrides `existingSecret`) | `nil` | +| `global.minio.accessKey` | MinIO Access Key (overrides `accessKey.password`) | `nil` | +| `global.minio.secretKey` | MinIO Secret Key (overrides `secretKey.password`) | `nil` | +| `image.registry` | MinIO image registry | `docker.io` | +| `image.repository` | MinIO image name | `bitnami/minio` | +| `image.tag` | MinIO image tag | `{TAG_NAME}` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug logs should be enabled | `false` | +| `nameOverride` | String to partially override minio.fullname template with a string (will prepend the release name) | `nil` | +| `fullnameOverride` | String to fully override minio.fullname template with a string | `nil` | +| `schedulerName` | Specifies the schedulerName, if it's nil uses kube-scheduler | `nil` | +| `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `serviceAccount.name` | If serviceAccount.create is enabled, what should the serviceAccount name be - otherwise defaults to the fullname | `nil` | +| `clusterDomain` | Kubernetes cluster domain | `cluster.local` | +| `clientImage.registry` | MinIO Client image registry | `docker.io` | +| `clientImage.repository` | MinIO Client image name | `bitnami/minio-client` | +| `clientImage.tag` | MinIO Client image tag | `{TAG_NAME}` | +| `volumePermissions.enabled` | Enable init container that changes volume permissions in the data directory (for cases where the default k8s `runAsUser` and `fsUser` values do not work) | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/minideb` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `buster` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.resources` | Init container resource requests/limit | `nil` | +| `mode` | MinIO server mode (`standalone` or `distributed`) | `standalone` | +| `statefulset.replicaCount` | Number of pods (only for Minio distributed mode). Should be 4 <= x <= 32 | `4` | +| `statefulset.updateStrategy` | Statefulset update strategy policy | `RollingUpdate` | +| `statefulset.podManagementpolicy` | Statefulset pods management policy | `Parallel` | +| `deployment.updateStrategy` | Deployment update strategy policy | `Recreate` | +| `existingSecret` | Existing secret with MinIO credentials | `nil` | +| `useCredentialsFile` | Have the secret mounted as a file instead of env vars | `false` | +| `forceNewKeys` | Force admin credentials (access and secret key) to be reconfigured every time they change in the secrets | `false` | +| `accessKey.password` | MinIO Access Key. Ignored if existing secret is provided. | _random 10 character alphanumeric string_ | +| `accessKey.forcePassword` | Force users to specify an Access Key | `false` | +| `secretKey.password` | MinIO Secret Key. Ignored if existing secret is provided. | _random 40 character alphanumeric string_ | +| `secretKey.forcePassword` | Force users to specify an Secret Key | `false` | +| `defaultBuckets` | Comma, semi-colon or space separated list of buckets to create (only in standalone mode) | `nil` | +| `disableWebUI` | Disable MinIO Web UI | `false` | +| `extraEnv` | Any extra environment variables you would like to pass to the pods | `{}` | +| `command` | Command for the minio container | `{}` | +| `resources` | Minio containers' resources | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `affinity` | Map of node/pod affinities | `{}` (The value is evaluated as a template) | +| `nodeSelector` | Node labels for pod assignment | `{}` (The value is evaluated as a template) | +| `tolerations` | Tolerations for pod assignment | `[]` (The value is evaluated as a template) | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the container | `1001` | +| `securityContext.runAsUser` | User ID for the container | `1001` | +| `livenessProbe.enabled` | Enable/disable the Liveness probe | `true` | +| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60` | +| `livenessProbe.periodSeconds` | How often to perform the probe | `10` | +| `livenessProbe.timeoutSeconds` | When the probe times out | `5` | +| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed. | `1` | +| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | `6` | +| `readinessProbe.enabled` | Enable/disable the Readiness probe | `true` | +| `readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `5` | +| `readinessProbe.periodSeconds` | How often to perform the probe | `10` | +| `readinessProbe.timeoutSeconds` | When the probe times out | `5` | +| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | `6` | +| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed. | `1` | +| `persistence.enabled` | Use a PVC to persist data | `true` | +| `persistence.mountPath` | Path to mount the volume at | `/data` | +| `persistence.storageClass` | Storage class of backing PVC | `nil` (uses alpha storage class annotation) | +| `persistence.accessMode` | Use volume as ReadOnly or ReadWrite | `ReadWriteOnce` | +| `persistence.size` | Size of data volume | `8Gi` | +| `persistence.annotations` | Persistent Volume annotations | `{}` | +| `persistence.existingClaim` | Name of an existing PVC to use (only in "standalone" mode) | `nil` | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | MinIO service port | `9000` | +| `service.nodePort` | Port to bind to for NodePort service type | `nil` | +| `service.loadBalancerIP` | Static IP Address to use for LoadBalancer service type | `nil` | +| `service.annotations` | Kubernetes service annotations | `{}` | +| `ingress.enabled` | Enable/disable ingress | `false` | +| `ingress.certManager` | Add annotations for cert-manager | `false` | +| `ingress.annotations` | Ingress annotations | `[]` | +| `ingress.labels` | Ingress additional labels | `{}` | +| `ingress.hosts[0].name` | Hostname to your MinIO installation | `minio.local` | +| `ingress.hosts[0].path` | Path within the url structure | `/` | +| `ingress.hosts[0].tls` | Utilize TLS backend in ingress | `false` | +| `ingress.hosts[0].tlsHosts` | Array of TLS hosts for ingress record (defaults to `ingress.hosts[0].name` if `nil`) | `nil` | +| `ingress.hosts[0].tlsSecret` | TLS Secret (certificates) | `minio.local-tls` | +| `ingress.secrets[0].name` | TLS Secret Name | `nil` | +| `ingress.secrets[0].certificate` | TLS Secret Certificate | `nil` | +| `ingress.secrets[0].key` | TLS Secret Key | `nil` | +| `networkPolicy.enabled` | Enable NetworkPolicy | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `prometheusAuthType` | Authentication mode for Prometheus (`jwt` or `public`) | `public` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install my-release \ + --set accessKey.password=minio-access-key \ + --set secretKey.password=minio-secret-key \ + bitnami/minio +``` + +The above command sets the MinIO Server access key and secret key to `minio-access-key` and `minio-secret-key`, respectively. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +$ helm install my-release -f values.yaml bitnami/minio +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Production configuration + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one. + +- MinIO server mode: +```diff +- mode: standalone ++ mode: distributed +``` + +- Disable MinIO Web UI: +```diff +- disableWebUI: false ++ disableWebUI: true +``` + +- Annotations to be added to pods: +```diff +- podAnnotations: {} ++ podAnnotations: ++ prometheus.io/scrape: "true" ++ prometheus.io/path: "/minio/prometheus/metrics" ++ prometheus.io/port: "9000" +``` + +- Pod resources: +```diff +- resources: {} ++ resources: ++ requests: ++ memory: 256Mi ++ cpu: 250m +``` + +- Enable NetworkPolicy: +```diff +- networkPolicy.enabled: false ++ networkPolicy.enabled: true +``` + +- Don't require client label for connections: +```diff +- networkPolicy.allowExternal: true ++ networkPolicy.allowExternal: false +``` + +- Change Prometheus authentication: +```diff +- prometheusAuthType: public ++ prometheusAuthType: jwt +``` + +### Distributed mode + +You can start the MinIO chart in distributed mode with the following parameter: `mode=distributed` + +This chart sets Minio server in distributed mode with 4 nodes by default. You can change the number of nodes setting the `statefulset.replicaCount` parameter, for example to `statefulset.replicaCount=8` + +> Note: that the number of replicas must even, greater than 4 and lower than 32 + +### Prometheus exporter + +MinIO exports Prometheus metrics at `/minio/prometheus/metrics`. To allow Prometheus collecting your MinIO metrics, modify the `values.yaml` adding the corresponding annotations: + +```diff +- podAnnotations: {} ++ podAnnotations: ++ prometheus.io/scrape: "true" ++ prometheus.io/path: "/minio/prometheus/metrics" ++ prometheus.io/port: "9000" +``` + +> Find more information about MinIO metrics at https://docs.min.io/docs/how-to-monitor-minio-using-prometheus.html + +## Persistence + +The [Bitnami MinIO](https://github.com/bitnami/bitnami-docker-minio) image stores data at the `/data` path of the container. + +The chart mounts a [Persistent Volume](http://kubernetes.io/docs/user-guide/persistent-volumes/) at this location. The volume is created using dynamic volume provisioning. + +### Adjust permissions of persistent volume mountpoint + +As the image run as non-root by default, it is necessary to adjust the ownership of the persistent volume so that the container can write data into it. + +By default, the chart is configured to use Kubernetes Security Context to automatically change the ownership of the volume. However, this feature does not work in all Kubernetes distributions. +As an alternative, this chart supports using an initContainer to change the ownership of the volume before mounting it in the final destination. + +You can enable this initContainer by setting `volumePermissions.enabled` to `true`. diff --git a/scripts/helm/helm/databases/charts/minio/ci/values-production.yaml b/scripts/helm/helm/databases/charts/minio/ci/values-production.yaml new file mode 100755 index 000000000..d5e966334 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/ci/values-production.yaml @@ -0,0 +1,27 @@ +# Test values file for generating all of the yaml and check that +# the rendering is correct + +volumePermissions: + enabled: true + +mode: distributed + +useCredentialsFile: true + +disableWebUI: false + +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/minio/prometheus/metric" + prometheus.io/port: "9000" + +resources: + requests: + cpu: 250m + memory: 256Mi + +ingress: + enabled: true + +networkPolicy: + enabled: true diff --git a/scripts/helm/helm/databases/charts/minio/templates/NOTES.txt b/scripts/helm/helm/databases/charts/minio/templates/NOTES.txt new file mode 100755 index 000000000..e7492fbe7 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/NOTES.txt @@ -0,0 +1,71 @@ +** Please be patient while the chart is being deployed ** + +MinIO can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ include "minio.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + +To get your credentials run: + + export MINIO_ACCESS_KEY=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "minio.fullname" . }} -o jsonpath="{.data.access-key}" | base64 --decode) + export MINIO_SECRET_KEY=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "minio.fullname" . }} -o jsonpath="{.data.secret-key}" | base64 --decode) + +To connect to your MinIO server using a client: + +- Run a MinIO Client pod and append the desired command (e.g. 'admin info server'): + + kubectl run --namespace {{ .Release.Namespace }} {{ include "minio.fullname" . }}-client \ + --rm --tty -i --restart='Never' \ + --env MINIO_SERVER_ACCESS_KEY=$MINIO_ACCESS_KEY \ + --env MINIO_SERVER_SECRET_KEY=$MINIO_SECRET_KEY \ + --env MINIO_SERVER_HOST={{ include "minio.fullname" . }} \ + {{- if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} + --labels="{{ template "minio.name" . }}-client=true" \ + {{- end }} + --image {{ template "minio.clientImage" . }} -- admin info server minio + +{{- if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} + + NOTE: Since NetworkPolicy is enabled, only pods with label + "{{ template "minio.fullname" . }}-client=true" will be able to connect to MinIO. +{{- end }} +{{- if not .Values.disableWebUI }} + +To access the MinIO web UI: + +- Get the MinIO URL: + +{{- if .Values.ingress.enabled }} + + You should be able to access your new MinIO web UI through + + {{- range .Values.ingress.hosts }} + {{ if .tls }}https{{ else }}http{{ end }}://{{ .name }}/minio/ + {{- end }} +{{- else if contains "LoadBalancer" .Values.service.type }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ include "minio.fullname" . }}' + + {{- $port:=.Values.service.port | toString }} + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "minio.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo "MinIO web URL: http://$SERVICE_IP{{- if ne $port "80" }}:{{ .Values.service.port }}{{ end }}/minio" + +{{- else if contains "ClusterIP" .Values.service.type }} + + echo "MinIO web URL: http://127.0.0.1:9000/minio" + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ include "minio.fullname" . }} 9000:{{ .Values.service.port }} + +{{- else if contains "NodePort" .Values.service.type }} + + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "minio.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo "MinIO web URL: http://$NODE_IP:$NODE_PORT/minio" + +{{- end }} +{{- else }} + + WARN: MinIO Web UI is disabled. +{{- end }} + +{{ include "minio.validateValues" . }} +{{ include "minio.checkRollingTags" . }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/_helpers.tpl b/scripts/helm/helm/databases/charts/minio/templates/_helpers.tpl new file mode 100755 index 000000000..dd67e53e6 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/_helpers.tpl @@ -0,0 +1,265 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Expand the name of the chart. +*/}} +{{- define "minio.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "minio.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minio.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "minio.labels" -}} +app.kubernetes.io/name: {{ include "minio.name" . }} +helm.sh/chart: {{ include "minio.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "minio.matchLabels" -}} +app.kubernetes.io/name: {{ include "minio.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Return the proper MinIO image name +*/}} +{{- define "minio.image" -}} +{{- $registryName := coalesce .Values.global.imageRegistry .Values.image.registry -}} +{{- $repositoryName := .Values.image.repository -}} +{{- $tag := .Values.image.tag | toString -}} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} + +{{/* +Return the proper MinIO Client image name +*/}} +{{- define "minio.clientImage" -}} +{{- $registryName := coalesce .Values.global.imageRegistry .Values.clientImage.registry -}} +{{- $repositoryName := .Values.clientImage.repository -}} +{{- $tag := .Values.clientImage.tag | toString -}} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "minio.imagePullSecrets" -}} +{{- $imagePullSecrets := coalesce .Values.global.imagePullSecrets .Values.image.pullSecrets .Values.volumePermissions.image.pullSecrets -}} +{{- if $imagePullSecrets }} +imagePullSecrets: +{{- range $imagePullSecrets }} + - name: {{ . }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return MinIO accessKey +*/}} +{{- define "minio.accessKey" -}} +{{- $accessKey := coalesce .Values.global.minio.accessKey .Values.accessKey.password -}} +{{- if $accessKey }} + {{- $accessKey -}} +{{- else if (not .Values.accessKey.forcePassword) }} + {{- randAlphaNum 10 -}} +{{- else -}} + {{ required "An Access Key is required!" .Values.accessKey.password }} +{{- end -}} +{{- end -}} + +{{/* +Return MinIO secretKey +*/}} +{{- define "minio.secretKey" -}} +{{- $secretKey := coalesce .Values.global.minio.secretKey .Values.secretKey.password -}} +{{- if $secretKey }} + {{- $secretKey -}} +{{- else if (not .Values.secretKey.forcePassword) }} + {{- randAlphaNum 40 -}} +{{- else -}} + {{ required "A Secret Key is required!" .Values.secretKey.password }} +{{- end -}} +{{- end -}} + +{{/* +Get the credentials secret. +*/}} +{{- define "minio.secretName" -}} +{{- if .Values.global.minio.existingSecret }} + {{- printf "%s" .Values.global.minio.existingSecret -}} +{{- else if .Values.existingSecret -}} + {{- printf "%s" .Values.existingSecret -}} +{{- else -}} + {{- printf "%s" (include "minio.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a secret object should be created +*/}} +{{- define "minio.createSecret" -}} +{{- if .Values.global.minio.existingSecret }} +{{- else if .Values.existingSecret -}} +{{- else -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Compile all warnings into a single message, and call fail. +*/}} +{{- define "minio.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "minio.validateValues.mode" .) -}} +{{- $messages := append $messages (include "minio.validateValues.replicaCount" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}} +{{- end -}} +{{- end -}} + +{{/* Validate values of MinIO - must provide a valid mode ("distributed" or "standalone") */}} +{{- define "minio.validateValues.mode" -}} +{{- if and (ne .Values.mode "distributed") (ne .Values.mode "standalone") -}} +minio: mode + Invalid mode selected. Valid values are "distributed" and + "standalone". Please set a valid mode (--set mode="xxxx") +{{- end -}} +{{- end -}} + +{{/* Validate values of MinIO - number of replicas must be even, greater than 4 and lower than 32 */}} +{{- define "minio.validateValues.replicaCount" -}} +{{- $replicaCount := int .Values.statefulset.replicaCount }} +{{- if and (eq .Values.mode "distributed") (or (eq (mod $replicaCount 2) 1) (lt $replicaCount 4) (gt $replicaCount 32)) -}} +minio: replicaCount + Number of replicas must be even, greater than 4 and lower than 32!! + Please set a valid number of replicas (--set statefulset.replicaCount=X) +{{- end -}} +{{- end -}} + +{{/* Check if there are rolling tags in the images */}} +{{- define "minio.checkRollingTags" -}} +{{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .Values.image.repository }}:{{ .Values.image.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} +{{- if and (contains "bitnami/" .Values.clientImage.repository) (not (.Values.clientImage.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .Values.clientImage.repository }}:{{ .Values.clientImage.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} +{{- end -}} + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "minio.volumePermissions.image" -}} +{{- $registryName := .Values.volumePermissions.image.registry -}} +{{- $repositoryName := .Values.volumePermissions.image.repository -}} +{{- $tag := .Values.volumePermissions.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Storage Class +*/}} +{{- define "minio.storageClass" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +*/}} +{{- if .Values.global -}} + {{- if .Values.global.storageClass -}} + {{- if (eq "-" .Values.global.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.global.storageClass -}} + {{- end -}} + {{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "minio.tplValue" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "minio.tplValue" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} + +{{/* +Returns the proper service account name depending if an explicit service account name is set +in the values file. If the name is not set it will default to either minio.fullname if serviceAccount.create +is true or default otherwise. +*/}} +{{- define "minio.serviceAccountName" -}} + {{- if .Values.serviceAccount.create -}} + {{ default (include "minio.fullname" .) .Values.serviceAccount.name }} + {{- else -}} + {{ default "default" .Values.serviceAccount.name }} + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/scripts/helm/helm/databases/charts/minio/templates/deployment-standalone.yaml b/scripts/helm/helm/databases/charts/minio/templates/deployment-standalone.yaml new file mode 100755 index 000000000..23a7232a8 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/deployment-standalone.yaml @@ -0,0 +1,160 @@ +{{- if eq .Values.mode "standalone" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +spec: + {{- if .Values.deployment.updateStrategy }} + strategy: {{ toYaml .Values.deployment.updateStrategy | nindent 4 }} + {{- end }} + selector: + matchLabels: {{- include "minio.matchLabels" . | nindent 6 }} + template: + metadata: + labels: {{- include "minio.labels" . | nindent 8 }} + {{- if or .Values.podAnnotations (include "minio.createSecret" .) }} + annotations: + {{- if (include "minio.createSecret" .) }} + checksum/credentials-secret: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- include "minio.tplValue" ( dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + serviceAccountName: {{ template "minio.serviceAccountName" . }} +{{- include "minio.imagePullSecrets" . | nindent 6 }} + {{- if .Values.affinity }} + affinity: {{- include "minio.tplValue" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "minio.tplValue" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "minio.tplValue" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + {{- end }} + {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }} + initContainers: + - name: volume-permissions + image: {{ template "minio.volumePermissions.image" . }} + imagePullPolicy: {{ default "" .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + chown -R {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.persistence.mountPath }} + securityContext: + runAsUser: 0 + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + containers: + - name: minio + image: {{ include "minio.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" .Values.image.debug | quote }} + - name: MINIO_FORCE_NEW_KEYS + value: {{ ternary "yes" "no" .Values.forceNewKeys | quote }} + {{- if .Values.useCredentialsFile }} + - name: MINIO_ACCESS_KEY_FILE + value: "/opt/bitnami/minio/secrets/access-key" + {{- else }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "minio.secretName" . }} + key: access-key + {{- end }} + {{- if .Values.useCredentialsFile }} + - name: MINIO_SECRET_KEY_FILE + value: "/opt/bitnami/minio/secrets/secret-key" + {{- else }} + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "minio.secretName" . }} + key: secret-key + {{- end }} + {{- if .Values.defaultBuckets }} + - name: MINIO_DEFAULT_BUCKETS + value: {{ .Values.defaultBuckets }} + {{- end }} + - name: MINIO_BROWSER + value: {{ ternary "off" "on" .Values.disableWebUI | quote }} + {{- if .Values.prometheusAuthType }} + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: {{ .Values.prometheusAuthType }} + {{- end }} + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 12 }} + {{- end }} + {{ if .Values.command }} + command: {{- toYaml .Values.command | nindent 12 }} + {{- end }} + ports: + - name: minio + containerPort: 9000 + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + path: /minio/health/live + port: minio + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + tcpSocket: + port: minio + {{- end }} + {{- if .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.useCredentialsFile }} + - name: minio-credentials + mountPath: /opt/bitnami/minio/secrets/ + {{- end }} + - name: "data" + mountPath: {{ .Values.persistence.mountPath }} + volumes: + {{- if .Values.useCredentialsFile }} + - name: minio-credentials + secret: + secretName: {{ include "minio.secretName" . }} + {{- end }} + - name: data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ include "minio.fullname" . }}{{- end }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/ingress.yaml b/scripts/helm/helm/databases/charts/minio/templates/ingress.yaml new file mode 100755 index 000000000..486feb5ac --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/ingress.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.ingress.enabled (not .Values.disableWebUI ) -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +{{- range $key, $value := .Values.ingress.labels }} + {{ $key }}: {{ $value }} +{{- end }} + annotations: + {{- if .Values.ingress.certManager }} + kubernetes.io/tls-acme: "true" + {{- end }} + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .name }} + http: + paths: + - path: {{ default "/" .path }} + backend: + serviceName: {{ include "minio.fullname" $ }} + servicePort: minio + {{- end }} + tls: + {{- range .Values.ingress.hosts }} + {{- if .tls }} + - hosts: + {{- if .tlsHosts }} + {{- range $host := .tlsHosts }} + - {{ $host }} + {{- end }} + {{- else }} + - {{ .name }} + {{- end }} + secretName: {{ .tlsSecret }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/networkpolicy.yaml b/scripts/helm/helm/databases/charts/minio/templates/networkpolicy.yaml new file mode 100755 index 000000000..cffc5be13 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/networkpolicy.yaml @@ -0,0 +1,23 @@ +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: {{ include "minio.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + ingress: + # Allow inbound connections + - ports: + - port: 9000 + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ include "minio.fullname" . }}-client: "true" + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/pvc-standalone.yaml b/scripts/helm/helm/databases/charts/minio/templates/pvc-standalone.yaml new file mode 100755 index 000000000..8e4be7da4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/pvc-standalone.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.mode "standalone") }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{ include "minio.storageClass" . }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/secrets.yaml b/scripts/helm/helm/databases/charts/minio/templates/secrets.yaml new file mode 100755 index 000000000..1a813eeda --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/secrets.yaml @@ -0,0 +1,12 @@ +{{- if (include "minio.createSecret" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +type: Opaque +data: + access-key: {{ include "minio.accessKey" . | b64enc | quote }} + secret-key: {{ include "minio.secretKey" . | b64enc | quote }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/service.yaml b/scripts/helm/helm/databases/charts/minio/templates/service.yaml new file mode 100755 index 000000000..7a6c380de --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} + {{- if .Values.service.annotations }} + annotations: {{- include "minio.tplValue" ( dict "value" .Values.service.annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP)) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: + - name: minio + port: {{ .Values.service.port }} + targetPort: minio + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePort)) }} + nodePort: {{ .Values.service.nodePort }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + selector: {{- include "minio.matchLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/serviceaccount.yaml b/scripts/helm/helm/databases/charts/minio/templates/serviceaccount.yaml new file mode 100755 index 000000000..76efe62f0 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "minio.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +secrets: + - name: {{ include "minio.fullname" . }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/statefulset.yaml b/scripts/helm/helm/databases/charts/minio/templates/statefulset.yaml new file mode 100755 index 000000000..7add4501f --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/statefulset.yaml @@ -0,0 +1,181 @@ +{{- if eq .Values.mode "distributed" }} +{{- $replicaCount := int .Values.statefulset.replicaCount }} +{{- if and (eq (mod $replicaCount 2) 0) (gt $replicaCount 3) (lt $replicaCount 33) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "minio.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +spec: + selector: + matchLabels: {{- include "minio.matchLabels" . | nindent 6 }} + serviceName: {{ include "minio.fullname" . }}-headless + replicas: {{ .Values.statefulset.replicaCount }} + podManagementPolicy: {{ .Values.statefulset.podManagementPolicy }} + updateStrategy: + type: {{ .Values.statefulset.updateStrategy }} + {{- if (eq "Recreate" .Values.statefulset.updateStrategy) }} + rollingUpdate: null + {{- end }} + template: + metadata: + labels: {{- include "minio.labels" . | nindent 8 }} + {{- if or .Values.podAnnotations (include "minio.createSecret" .) }} + annotations: + {{- if (include "minio.createSecret" .) }} + checksum/credentials-secret: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- include "minio.tplValue" ( dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName }} + {{- end }} + serviceAccountName: {{ template "minio.serviceAccountName" . }} +{{- include "minio.imagePullSecrets" . | nindent 6 }} + {{- if .Values.affinity }} + affinity: {{- include "minio.tplValue" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "minio.tplValue" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "minio.tplValue" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + {{- end }} + {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }} + initContainers: + - name: volume-permissions + image: {{ template "minio.volumePermissions.image" . }} + imagePullPolicy: {{ default "" .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + chown -R {{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.persistence.mountPath }} + securityContext: + runAsUser: 0 + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + containers: + - name: minio + image: {{ include "minio.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" .Values.image.debug | quote }} + - name: MINIO_DISTRIBUTED_MODE_ENABLED + value: "yes" + - name: MINIO_SKIP_CLIENT + value: "yes" + - name: MINIO_DISTRIBUTED_NODES + {{- $minioFullname := include "minio.fullname" . }} + {{- $minioHeadlessServiceName := printf "%s-%s" $minioFullname "headless" | trunc 63 }} + {{- $releaseNamespace := .Release.Namespace }} + value: {{range $i, $e := until $replicaCount }}{{ $minioFullname }}-{{ $e }}.{{ $minioHeadlessServiceName }}.{{ $releaseNamespace }}.svc.{{ $.Values.clusterDomain }},{{ end }} + - name: MINIO_FORCE_NEW_KEYS + value: {{ ternary "yes" "no" .Values.forceNewKeys | quote }} + {{- if .Values.useCredentialsFile }} + - name: MINIO_ACCESS_KEY_FILE + value: "/opt/bitnami/minio/secrets/access-key" + {{- else }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "minio.secretName" . }} + key: access-key + {{- end }} + {{- if .Values.useCredentialsFile }} + - name: MINIO_SECRET_KEY_FILE + value: "/opt/bitnami/minio/secrets/secret-key" + {{- else }} + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "minio.secretName" . }} + key: secret-key + {{- end }} + - name: MINIO_BROWSER + value: {{ ternary "off" "on" .Values.disableWebUI | quote }} + {{- if .Values.prometheusAuthType }} + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: {{ .Values.prometheusAuthType }} + {{- end }} + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 12 }} + {{- end }} + ports: + - name: minio + containerPort: 9000 + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + httpGet: + path: /minio/health/live + port: minio + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + tcpSocket: + port: minio + {{- end }} + {{- if .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.useCredentialsFile }} + - name: minio-credentials + mountPath: /opt/bitnami/minio/secrets/ + {{- end }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + volumes: + {{- if .Values.useCredentialsFile }} + - name: minio-credentials + secret: + secretName: {{ include "minio.secretName" . }} + {{- end }} +{{- if not .Values.persistence.enabled }} + - name: data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + labels: {{- include "minio.matchLabels" . | nindent 10 }} + {{- if .Values.persistence.annotations }} + annotations: {{- include "minio.tplValue" ( dict "value" .Values.persistence.annotations "context" $) | nindent 10 }} + {{- end }} + spec: + accessModes: {{ toYaml .Values.persistence.accessModes | nindent 10 }} + {{ include "minio.storageClass" . }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/svc-headless.yaml b/scripts/helm/helm/databases/charts/minio/templates/svc-headless.yaml new file mode 100755 index 000000000..41a4cf507 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/svc-headless.yaml @@ -0,0 +1,16 @@ +{{- if eq .Values.mode "distributed" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "minio.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: minio + port: {{ .Values.service.port }} + targetPort: minio + selector: {{- include "minio.matchLabels" . | nindent 4 }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/templates/tls-secrets.yaml b/scripts/helm/helm/databases/charts/minio/templates/tls-secrets.yaml new file mode 100755 index 000000000..68b31d324 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/templates/tls-secrets.yaml @@ -0,0 +1,14 @@ +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "minio.labels" $ | nindent 4 }} +type: kubernetes.io/tls +data: + tls.crt: {{ .certificate | b64enc }} + tls.key: {{ .key | b64enc }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/minio/values-production.yaml b/scripts/helm/helm/databases/charts/minio/values-production.yaml new file mode 100755 index 000000000..d4d639732 --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/values-production.yaml @@ -0,0 +1,389 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + minio: {} +# minio: +# existingSecret: "" +# accessKey: "" +# secretKey: "" +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami MinIO image version +## ref: https://hub.docker.com/r/bitnami/minio/tags/ +## +image: + registry: docker.io + repository: bitnami/minio + tag: 2020.10.9-debian-10-r6 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## String to partially override minio.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override minio.fullname template +## +# fullnameOverride: + +## Scheduler name +## https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: stork + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + +## Cluster domain +## +clusterDomain: cluster.local + +## Bitnami MinIO Client image version +## ref: https://hub.docker.com/r/bitnami/minio-client/tags/ +## +clientImage: + registry: docker.io + repository: bitnami/minio-client + tag: 2020.10.3-debian-10-r9 + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Init container' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## MinIO server mode. Allowed values: standalone or distributed. +## Distributed Minio ref: https://docs.minio.io/docs/distributed-minio-quickstart-guide +## +mode: distributed + +## MinIO deployment parameters +## Only when mode is 'standalone' +## +deployment: + ## Set to Recreate if you use persistent volume that cannot be mounted by more than one pods to makesure the pods is destroyed first. + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + ## Example: + ## updateStrategy: + ## type: RollingUpdate + ## rollingUpdate: + ## maxSurge: 25% + ## maxUnavailable: 25% + ## + updateStrategy: + type: Recreate + +## MinIO statefulset parameters +## Only when mode is 'distributed' +## +statefulset: + ## Update strategy, can be set to RollingUpdate or OnDelete by default. + ## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + ## + updateStrategy: RollingUpdate + + ## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel + ## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy + ## + podManagementPolicy: Parallel + + ## Number of replicas, it must even and greater than 4 + ## + replicaCount: 4 + +## Use existing secret (ignores accessKey, and secretKey passwords) +## +# existingSecret: + +## Mount MinIO secret as a file instead of passing environment variable +## +useCredentialsFile: false + +## Force reconfiguring new keys whenever the credentials change +## +forceNewKeys: false + +## MinIO credentials +## +accessKey: + ## MinIO Access Key + ## ref: https://github.com/bitnami/bitnami-docker-minio/#setting-up-minio-in-distributed-mode + ## + password: + ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly. + ## If it is not force, a random password will be generated. + ## + forcePassword: false +secretKey: + ## MinIO Secret Key + ## ref: https://github.com/bitnami/bitnami-docker-minio/#setting-up-minio-in-distributed-mode + ## + password: + ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly. + ## If it is not force, a random password will be generated. + ## + forcePassword: false + +## Comma, semi-colon or space separated list of buckets to create at initialization (only in standalone mode) +## +# defaultBuckets: "my-bucket, my-second-bucket" + +## Disable MinIO Web UI +## ref: https://github.com/minio/minio/tree/master/docs/config/#browser +## +disableWebUI: true + +## Define custom environment variables to pass to the image here +## +extraEnv: {} + +## Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Pod annotations +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/minio/prometheus/metric" + prometheus.io/port: "9000" + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template. +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## MinIO containers' resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 250m + # memory: 256Mi + requests: + cpu: 250m + memory: 256Mi + +## MinIO containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes +## +livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + ## If true, use a Persistent Volume Claim, If false, use emptyDir + ## + enabled: true + + ## Enable persistence using an existing PVC (only in standalone mode) + ## + # existingClaim: + + ## Data volume mount path + ## + mountPath: /data + + ## Persistent Volume Access Mode + ## + accessModes: + - ReadWriteOnce + + ## Persistent Volume size + ## + size: 8Gi + + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + + ## Persistent Volume Claim annotations + ## + annotations: {} + +## MinIO Service properties +## +service: + ## MinIO Service type + ## + type: ClusterIP + + ## MinIO Service port + ## + port: 9000 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + + ## loadBalancerIP for the PrestaShop Service (optional, cloud specific) + ## ref: http://kubernetes.io/docs/user-guide/services/#type-loadbalancer + ## + # loadBalancerIP: + +## Configure the ingress resource that allows you to access the +## MinIO web browser. Set up the URL +## ref: http://kubernetes.io/docs/user-guide/ingress/ +## +ingress: + ## Set to true to enable ingress record generation + enabled: false + + ## Set this to true in order to add the corresponding annotations for cert-manager + certManager: false + + ## Ingress annotations done as key:value pairs + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set + # annotations: + # kubernetes.io/ingress.class: nginx + + ## Ingress additional labels done as key:value pairs + labels: {} + + ## The list of hostnames to be covered with this ingress record. + ## Most likely this will be just one host, but in the event more hosts are needed, this is an array + hosts: + - name: minio.local + path: / + + ## Set this to true in order to enable TLS on the ingress record + tls: false + + ## Optionally specify the TLS hosts for the ingress record + ## Useful when the Ingress controller supports www-redirection + ## If not specified, the above host name will be used + # tlsHosts: + # - www.minio.local + # - minio.local + + ## If TLS is set to true, you must declare what secret will store the key/certificate for TLS + tlsSecret: minio.local-tls + + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using cert-manager, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: minio.local-tls + # key: + # certificate: + +## NetworkPolicy parameters +## +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: true + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port MinIO is listening + ## on. When true, MinIO will accept connections from any source + ## (with the correct destination port). + ## + allowExternal: false + +## MinIO supports two authentication modes for Prometheus either jwt or public, by default MinIO runs in jwt mode. +## To allow public access without authentication for prometheus metrics set environment as follows. +prometheusAuthType: jwt diff --git a/scripts/helm/helm/databases/charts/minio/values.yaml b/scripts/helm/helm/databases/charts/minio/values.yaml new file mode 100755 index 000000000..8aee06beb --- /dev/null +++ b/scripts/helm/helm/databases/charts/minio/values.yaml @@ -0,0 +1,391 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + minio: {} +# accessKey: "minios3AccessKeyS3cr3t" +# secretKey: "m1n10s3CretK3yPassw0rd" +# minio: {} +# existingSecret: "" +# accessKey: "" +# secretKey: "" +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami MinIO image version +## ref: https://hub.docker.com/r/bitnami/minio/tags/ +## +image: + registry: docker.io + repository: bitnami/minio + tag: 2020.10.9-debian-10-r6 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + debug: false + +## String to partially override minio.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override minio.fullname template +## +# fullnameOverride: + +## Scheduler name +## https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: stork + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + +## Cluster domain +## +clusterDomain: cluster.local + +## Bitnami MinIO Client image version +## ref: https://hub.docker.com/r/bitnami/minio-client/tags/ +## +clientImage: + registry: docker.io + repository: bitnami/minio-client + tag: 2020.10.3-debian-10-r9 + +## Init containers parameters: +## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Init container' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## MinIO server mode. Allowed values: standalone or distributed. +## Distributed Minio ref: https://docs.minio.io/docs/distributed-minio-quickstart-guide +## +mode: standalone + +## MinIO deployment parameters +## Only when mode is 'standalone' +## +deployment: + ## Set to Recreate if you use persistent volume that cannot be mounted by more than one pods to makesure the pods is destroyed first. + ## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + ## Example: + ## updateStrategy: + ## type: RollingUpdate + ## rollingUpdate: + ## maxSurge: 25% + ## maxUnavailable: 25% + ## + updateStrategy: + type: Recreate + +## MinIO statefulset parameters +## Only when mode is 'distributed' +## +statefulset: + ## Update strategy, can be set to RollingUpdate or OnDelete by default. + ## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + ## + updateStrategy: RollingUpdate + + ## StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel + ## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy + ## + podManagementPolicy: Parallel + + ## Number of replicas, it must even and greater than 4 + ## + replicaCount: 4 + +## Use existing secret (ignores accessKey, and secretKey passwords) +## +# existingSecret: + +## Mount MinIO secret as a file instead of passing environment variable +## +useCredentialsFile: false + +## Force reconfiguring new keys whenever the credentials change +## +forceNewKeys: false + +## MinIO credentials +## +accessKey: + ## MinIO Access Key + ## ref: https://github.com/bitnami/bitnami-docker-minio/#setting-up-minio-in-distributed-mode + ## + password: + ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly. + ## If it is not force, a random password will be generated. + ## + forcePassword: false +secretKey: + ## MinIO Secret Key + ## ref: https://github.com/bitnami/bitnami-docker-minio/#setting-up-minio-in-distributed-mode + ## + password: + ## Option to force users to specify a password. That is required for 'helm upgrade' to work properly. + ## If it is not force, a random password will be generated. + ## + forcePassword: false + +## Comma, semi-colon or space separated list of buckets to create at initialization (only in standalone mode) +## +# defaultBuckets: "my-bucket, my-second-bucket" + +## Disable MinIO Web UI +## ref: https://github.com/minio/minio/tree/master/docs/config/#browser +## +disableWebUI: false + +## Define custom environment variables to pass to the image here +## +extraEnv: {} + +## Define a custom command for the minio container +command: {} + +## Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + +## Pod annotations +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} + +## Affinity for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Node labels for pod assignment. Evaluated as a template. +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## MinIO containers' resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 250m + # memory: 256Mi + requests: {} + # cpu: 250m + # memory: 256Mi + +## MinIO containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes +## +livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + ## If true, use a Persistent Volume Claim, If false, use emptyDir + ## + enabled: true + + ## Enable persistence using an existing PVC (only in standalone mode) + ## + # existingClaim: + + ## Data volume mount path + ## + mountPath: /data + + ## Persistent Volume Access Mode + ## + accessModes: + - ReadWriteOnce + + ## Persistent Volume size + ## + size: 50Gi + + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + + ## Persistent Volume Claim annotations + ## + annotations: {} + +## MinIO Service properties +## +service: + ## MinIO Service type + ## + type: ClusterIP + + ## MinIO Service port + ## + port: 9000 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + + ## loadBalancerIP for the PrestaShop Service (optional, cloud specific) + ## ref: http://kubernetes.io/docs/user-guide/services/#type-loadbalancer + ## + # loadBalancerIP: + +## Configure the ingress resource that allows you to access the +## MinIO web browser. Set up the URL +## ref: http://kubernetes.io/docs/user-guide/ingress/ +## +ingress: + ## Set to true to enable ingress record generation + enabled: false + + ## Set this to true in order to add the corresponding annotations for cert-manager + certManager: false + + ## Ingress annotations done as key:value pairs + ## For a full list of possible ingress annotations, please see + ## ref: https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/annotations.md + ## + ## If tls is set to true, annotation ingress.kubernetes.io/secure-backends: "true" will automatically be set + ## If certManager is set to true, annotation kubernetes.io/tls-acme: "true" will automatically be set + # annotations: + # kubernetes.io/ingress.class: nginx + + ## Ingress additional labels done as key:value pairs + labels: {} + + ## The list of hostnames to be covered with this ingress record. + ## Most likely this will be just one host, but in the event more hosts are needed, this is an array + hosts: + - name: minio.local + path: / + + ## Set this to true in order to enable TLS on the ingress record + tls: false + + ## Optionally specify the TLS hosts for the ingress record + ## Useful when the Ingress controller supports www-redirection + ## If not specified, the above host name will be used + # tlsHosts: + # - www.minio.local + # - minio.local + + ## If TLS is set to true, you must declare what secret will store the key/certificate for TLS + tlsSecret: minio.local-tls + + secrets: + ## If you're providing your own certificates, please use this to add the certificates as secrets + ## key and certificate should start with -----BEGIN CERTIFICATE----- or + ## -----BEGIN RSA PRIVATE KEY----- + ## + ## name should line up with a tlsSecret set further up + ## If you're using cert-manager, this is unneeded, as it will create the secret for you if it is not set + ## + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + # - name: minio.local-tls + # key: + # certificate: + +## NetworkPolicy parameters +## +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port MinIO is listening + ## on. When true, MinIO will accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + +## MinIO supports two authentication modes for Prometheus either jwt or public, by default MinIO runs in jwt mode. +## To allow public access without authentication for prometheus metrics set environment as follows. +prometheusAuthType: public diff --git a/scripts/helm/helm/databases/charts/postgresql/.helmignore b/scripts/helm/helm/databases/charts/postgresql/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/scripts/helm/helm/databases/charts/postgresql/Chart.yaml b/scripts/helm/helm/databases/charts/postgresql/Chart.yaml new file mode 100755 index 000000000..3ac2d3605 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/Chart.yaml @@ -0,0 +1,26 @@ +annotations: + category: Database +apiVersion: v1 +appVersion: 11.9.0 +description: Chart for PostgreSQL, an object-relational database management system + (ORDBMS) with an emphasis on extensibility and on standards-compliance. +engine: gotpl +home: https://github.com/bitnami/charts/tree/master/bitnami/postgresql +icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-110x117.png +keywords: +- postgresql +- postgres +- database +- sql +- replication +- cluster +maintainers: +- email: containers@bitnami.com + name: Bitnami +- email: cedric@desaintmartin.fr + name: desaintmartin +name: postgresql +sources: +- https://github.com/bitnami/bitnami-docker-postgresql +- https://www.postgresql.org/ +version: 9.8.2 diff --git a/scripts/helm/helm/databases/charts/postgresql/README.md b/scripts/helm/helm/databases/charts/postgresql/README.md new file mode 100755 index 000000000..8cdb2ca13 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/README.md @@ -0,0 +1,707 @@ +# PostgreSQL + +[PostgreSQL](https://www.postgresql.org/) is an object-relational database management system (ORDBMS) with an emphasis on extensibility and on standards-compliance. + +For HA, please see [this repo](https://github.com/bitnami/charts/tree/master/bitnami/postgresql-ha) + +## TL;DR + +```console +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/postgresql +``` + +## Introduction + +This chart bootstraps a [PostgreSQL](https://github.com/bitnami/bitnami-docker-postgresql) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This chart has been tested to work with NGINX Ingress, cert-manager, fluentd and Prometheus on top of the [BKPR](https://kubeprod.io/). + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart +To install the chart with the release name `my-release`: + +```console +$ helm install my-release bitnami/postgresql +``` + +The command deploys PostgreSQL on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components but PVC's associated with the chart and deletes the release. + +To delete the PVC's associated with `my-release`: + +```console +$ kubectl delete pvc -l release=my-release +``` + +> **Note**: Deleting the PVC's will delete postgresql data as well. Please be cautious before doing it. + +## Parameters + +The following tables lists the configurable parameters of the PostgreSQL chart and their default values. + +| Parameter | Description | Default | +|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------| +| `global.imageRegistry` | Global Docker Image registry | `nil` | +| `global.postgresql.postgresqlDatabase` | PostgreSQL database (overrides `postgresqlDatabase`) | `nil` | +| `global.postgresql.postgresqlUsername` | PostgreSQL username (overrides `postgresqlUsername`) | `nil` | +| `global.postgresql.existingSecret` | Name of existing secret to use for PostgreSQL passwords (overrides `existingSecret`) | `nil` | +| `global.postgresql.postgresqlPassword` | PostgreSQL admin password (overrides `postgresqlPassword`) | `nil` | +| `global.postgresql.servicePort` | PostgreSQL port (overrides `service.port`) | `nil` | +| `global.postgresql.replicationPassword` | Replication user password (overrides `replication.password`) | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | +| `image.registry` | PostgreSQL Image registry | `docker.io` | +| `image.repository` | PostgreSQL Image name | `bitnami/postgresql` | +| `image.tag` | PostgreSQL Image tag | `{TAG_NAME}` | +| `image.pullPolicy` | PostgreSQL Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify Image pull secrets | `nil` (does not add image pull secrets to deployed pods) | +| `image.debug` | Specify if debug values should be set | `false` | +| `nameOverride` | String to partially override postgresql.fullname template with a string (will prepend the release name) | `nil` | +| `fullnameOverride` | String to fully override postgresql.fullname template with a string | `nil` | +| `volumePermissions.enabled` | Enable init container that changes volume permissions in the data directory (for cases where the default k8s `runAsUser` and `fsUser` values do not work) | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/minideb` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `buster` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.securityContext.*` | Other container security context to be included as-is in the container spec | `{}` | +| `volumePermissions.securityContext.runAsUser` | User ID for the init container (when facing issues in OpenShift or uid unknown, try value "auto") | `0` | +| `usePasswordFile` | Have the secrets mounted as a file instead of env vars | `false` | +| `ldap.enabled` | Enable LDAP support | `false` | +| `ldap.existingSecret` | Name of existing secret to use for LDAP passwords | `nil` | +| `ldap.url` | LDAP URL beginning in the form `ldap[s]://host[:port]/basedn[?[attribute][?[scope][?[filter]]]]` | `nil` | +| `ldap.server` | IP address or name of the LDAP server. | `nil` | +| `ldap.port` | Port number on the LDAP server to connect to | `nil` | +| `ldap.scheme` | Set to `ldaps` to use LDAPS. | `nil` | +| `ldap.tls` | Set to `1` to use TLS encryption | `nil` | +| `ldap.prefix` | String to prepend to the user name when forming the DN to bind | `nil` | +| `ldap.suffix` | String to append to the user name when forming the DN to bind | `nil` | +| `ldap.search_attr` | Attribute to match agains the user name in the search | `nil` | +| `ldap.search_filter` | The search filter to use when doing search+bind authentication | `nil` | +| `ldap.baseDN` | Root DN to begin the search for the user in | `nil` | +| `ldap.bindDN` | DN of user to bind to LDAP | `nil` | +| `ldap.bind_password` | Password for the user to bind to LDAP | `nil` | +| `replication.enabled` | Enable replication | `false` | +| `replication.user` | Replication user | `repl_user` | +| `replication.password` | Replication user password | `repl_password` | +| `replication.slaveReplicas` | Number of slaves replicas | `1` | +| `replication.synchronousCommit` | Set synchronous commit mode. Allowed values: `on`, `remote_apply`, `remote_write`, `local` and `off` | `off` | +| `replication.numSynchronousReplicas` | Number of replicas that will have synchronous replication. Note: Cannot be greater than `replication.slaveReplicas`. | `0` | +| `replication.applicationName` | Cluster application name. Useful for advanced replication settings | `my_application` | +| `existingSecret` | Name of existing secret to use for PostgreSQL passwords. The secret has to contain the keys `postgresql-password` which is the password for `postgresqlUsername` when it is different of `postgres`, `postgresql-postgres-password` which will override `postgresqlPassword`, `postgresql-replication-password` which will override `replication.password` and `postgresql-ldap-password` which will be sed to authenticate on LDAP. The value is evaluated as a template. | `nil` | +| `postgresqlPostgresPassword` | PostgreSQL admin password (used when `postgresqlUsername` is not `postgres`, in which case`postgres` is the admin username). | _random 10 character alphanumeric string_ | +| `postgresqlUsername` | PostgreSQL user (creates a non-admin user when `postgresqlUsername` is not `postgres`) | `postgres` | +| `postgresqlPassword` | PostgreSQL user password | _random 10 character alphanumeric string_ | +| `postgresqlDatabase` | PostgreSQL database | `nil` | +| `postgresqlDataDir` | PostgreSQL data dir folder | `/bitnami/postgresql` (same value as persistence.mountPath) | +| `extraEnv` | Any extra environment variables you would like to pass on to the pod. The value is evaluated as a template. | `[]` | +| `extraEnvVarsCM` | Name of a Config Map containing extra environment variables you would like to pass on to the pod. The value is evaluated as a template. | `nil` | +| `postgresqlInitdbArgs` | PostgreSQL initdb extra arguments | `nil` | +| `postgresqlInitdbWalDir` | PostgreSQL location for transaction log | `nil` | +| `postgresqlConfiguration` | Runtime Config Parameters | `nil` | +| `postgresqlExtendedConf` | Extended Runtime Config Parameters (appended to main or default configuration) | `nil` | +| `pgHbaConfiguration` | Content of pg_hba.conf | `nil (do not create pg_hba.conf)` | +| `postgresqlSharedPreloadLibraries` | Shared preload libraries (comma-separated list) | `pgaudit` | +| `postgresqlMaxConnections` | Maximum total connections | `nil` | +| `postgresqlPostgresConnectionLimit` | Maximum total connections for the postgres user | `nil` | +| `postgresqlDbUserConnectionLimit` | Maximum total connections for the non-admin user | `nil` | +| `postgresqlTcpKeepalivesInterval` | TCP keepalives interval | `nil` | +| `postgresqlTcpKeepalivesIdle` | TCP keepalives idle | `nil` | +| `postgresqlTcpKeepalivesCount` | TCP keepalives count | `nil` | +| `postgresqlStatementTimeout` | Statement timeout | `nil` | +| `postgresqlPghbaRemoveFilters` | Comma-separated list of patterns to remove from the pg_hba.conf file | `nil` | +| `customLivenessProbe` | Override default liveness probe | `nil` | +| `customReadinessProbe` | Override default readiness probe | `nil` | +| `audit.logHostname` | Add client hostnames to the log file | `false` | +| `audit.logConnections` | Add client log-in operations to the log file | `false` | +| `audit.logDisconnections` | Add client log-outs operations to the log file | `false` | +| `audit.pgAuditLog` | Add operations to log using the pgAudit extension | `nil` | +| `audit.clientMinMessages` | Message log level to share with the user | `nil` | +| `audit.logLinePrefix` | Template string for the log line prefix | `nil` | +| `audit.logTimezone` | Timezone for the log timestamps | `nil` | +| `configurationConfigMap` | ConfigMap with the PostgreSQL configuration files (Note: Overrides `postgresqlConfiguration` and `pgHbaConfiguration`). The value is evaluated as a template. | `nil` | +| `extendedConfConfigMap` | ConfigMap with the extended PostgreSQL configuration files. The value is evaluated as a template. | `nil` | +| `initdbScripts` | Dictionary of initdb scripts | `nil` | +| `initdbUser` | PostgreSQL user to execute the .sql and sql.gz scripts | `nil` | +| `initdbPassword` | Password for the user specified in `initdbUser` | `nil` | +| `initdbScriptsConfigMap` | ConfigMap with the initdb scripts (Note: Overrides `initdbScripts`). The value is evaluated as a template. | `nil` | +| `initdbScriptsSecret` | Secret with initdb scripts that contain sensitive information (Note: can be used with `initdbScriptsConfigMap` or `initdbScripts`). The value is evaluated as a template. | `nil` | +| `service.type` | Kubernetes Service type | `ClusterIP` | +| `service.port` | PostgreSQL port | `5432` | +| `service.nodePort` | Kubernetes Service nodePort | `nil` | +| `service.annotations` | Annotations for PostgreSQL service | `{}` (evaluated as a template) | +| `service.loadBalancerIP` | loadBalancerIP if service type is `LoadBalancer` | `nil` | +| `service.loadBalancerSourceRanges` | Address that are allowed when svc is LoadBalancer | `[]` (evaluated as a template) | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `shmVolume.enabled` | Enable emptyDir volume for /dev/shm for master and slave(s) Pod(s) | `true` | +| `shmVolume.chmod.enabled` | Run at init chmod 777 of the /dev/shm (ignored if `volumePermissions.enabled` is `false`) | `true` | +| `persistence.enabled` | Enable persistence using PVC | `true` | +| `persistence.existingClaim` | Provide an existing `PersistentVolumeClaim`, the value is evaluated as a template. | `nil` | +| `persistence.mountPath` | Path to mount the volume at | `/bitnami/postgresql` | +| `persistence.subPath` | Subdirectory of the volume to mount at | `""` | +| `persistence.storageClass` | PVC Storage Class for PostgreSQL volume | `nil` | +| `persistence.accessModes` | PVC Access Mode for PostgreSQL volume | `[ReadWriteOnce]` | +| `persistence.size` | PVC Storage Request for PostgreSQL volume | `8Gi` | +| `persistence.annotations` | Annotations for the PVC | `{}` | +| `commonAnnotations` | Annotations to be added to all deployed resources (rendered as a template) | `{}` | +| `master.nodeSelector` | Node labels for pod assignment (postgresql master) | `{}` | +| `master.affinity` | Affinity labels for pod assignment (postgresql master) | `{}` | +| `master.tolerations` | Toleration labels for pod assignment (postgresql master) | `[]` | +| `master.anotations` | Map of annotations to add to the statefulset (postgresql master) | `{}` | +| `master.labels` | Map of labels to add to the statefulset (postgresql master) | `{}` | +| `master.podAnnotations` | Map of annotations to add to the pods (postgresql master) | `{}` | +| `master.podLabels` | Map of labels to add to the pods (postgresql master) | `{}` | +| `master.priorityClassName` | Priority Class to use for each pod (postgresql master) | `nil` | +| `master.extraInitContainers` | Additional init containers to add to the pods (postgresql master) | `[]` | +| `master.extraVolumeMounts` | Additional volume mounts to add to the pods (postgresql master) | `[]` | +| `master.extraVolumes` | Additional volumes to add to the pods (postgresql master) | `[]` | +| `master.sidecars` | Add additional containers to the pod | `[]` | +| `master.service.type` | Allows using a different service type for Master | `nil` | +| `master.service.nodePort` | Allows using a different nodePort for Master | `nil` | +| `master.service.clusterIP` | Allows using a different clusterIP for Master | `nil` | +| `masterAsStandBy.enabled` | Whether to enable current cluster's Master as standby server of another cluster or not. | `false` | +| `masterAsStandBy.masterHost` | The Host of replication Master in the other cluster. | `nil` | +| `masterAsStandBy.masterPort ` | The Port of replication Master in the other cluster. | `nil` | +| `slave.nodeSelector` | Node labels for pod assignment (postgresql slave) | `{}` | +| `slave.affinity` | Affinity labels for pod assignment (postgresql slave) | `{}` | +| `slave.tolerations` | Toleration labels for pod assignment (postgresql slave) | `[]` | +| `slave.anotations` | Map of annotations to add to the statefulsets (postgresql slave) | `{}` | +| `slave.resources` | CPU/Memory resource requests/limits override for slaves. Will fallback to `values.resources` if not defined. | `{}` | +| `slave.labels` | Map of labels to add to the statefulsets (postgresql slave) | `{}` | +| `slave.podAnnotations` | Map of annotations to add to the pods (postgresql slave) | `{}` | +| `slave.podLabels` | Map of labels to add to the pods (postgresql slave) | `{}` | +| `slave.priorityClassName` | Priority Class to use for each pod (postgresql slave) | `nil` | +| `slave.extraInitContainers` | Additional init containers to add to the pods (postgresql slave) | `[]` | +| `slave.extraVolumeMounts` | Additional volume mounts to add to the pods (postgresql slave) | `[]` | +| `slave.extraVolumes` | Additional volumes to add to the pods (postgresql slave) | `[]` | +| `slave.sidecars` | Add additional containers to the pod | `[]` | +| `slave.service.type` | Allows using a different service type for Slave | `nil` | +| `slave.service.nodePort` | Allows using a different nodePort for Slave | `nil` | +| `slave.service.clusterIP` | Allows using a different clusterIP for Slave | `nil` | +| `slave.persistence.enabled` | Whether to enable slave replicas persistence | `true` | +| `terminationGracePeriodSeconds` | Seconds the pod needs to terminate gracefully | `nil` | +| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `securityContext.*` | Other pod security context to be included as-is in the pod spec | `{}` | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the pod | `1001` | +| `containerSecurityContext.*` | Other container security context to be included as-is in the container spec | `{}` | +| `containerSecurityContext.enabled` | Enable container security context | `true` | +| `containerSecurityContext.runAsUser` | User ID for the container | `1001` | +| `serviceAccount.enabled` | Enable service account (Note: Service Account will only be automatically created if `serviceAccount.name` is not set) | `false` | +| `serviceAccount.name` | Name of existing service account | `nil` | +| `livenessProbe.enabled` | Would you like a livenessProbe to be enabled | `true` | +| `networkPolicy.enabled` | Enable NetworkPolicy | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which ingress traffic could be allowed | `{}` | +| `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | +| `livenessProbe.periodSeconds` | How often to perform the probe | 10 | +| `livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | +| `readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | 5 | +| `readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `tls.enabled` | Enable TLS traffic support | `false` | +| `tls.preferServerCiphers` | Whether to use the server's TLS cipher preferences rather than the client's | `true` | +| `tls.certificatesSecret` | Name of an existing secret that contains the certificates | `nil` | +| `tls.certFilename` | Certificate filename | `""` | +| `tls.certKeyFilename` | Certificate key filename | `""` | +| `tls.certCAFilename` | CA Certificate filename. If provided, PostgreSQL will authenticate TLS/SSL clients by requesting them a certificate. | `nil` | +| `tls.crlFilename` | File containing a Certificate Revocation List | `nil` | +| `metrics.enabled` | Start a prometheus exporter | `false` | +| `metrics.service.type` | Kubernetes Service type | `ClusterIP` | +| `service.clusterIP` | Static clusterIP or None for headless services | `nil` | +| `metrics.service.annotations` | Additional annotations for metrics exporter pod | `{ prometheus.io/scrape: "true", prometheus.io/port: "9187"}` | +| `metrics.service.loadBalancerIP` | loadBalancerIP if redis metrics service type is `LoadBalancer` | `nil` | +| `metrics.serviceMonitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `metrics.serviceMonitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | +| `metrics.serviceMonitor.namespace` | Optional namespace in which to create ServiceMonitor | `nil` | +| `metrics.serviceMonitor.interval` | Scrape interval. If not set, the Prometheus default scrape interval is used | `nil` | +| `metrics.serviceMonitor.scrapeTimeout` | Scrape timeout. If not set, the Prometheus default scrape timeout is used | `nil` | +| `metrics.prometheusRule.enabled` | Set this to true to create prometheusRules for Prometheus operator | `false` | +| `metrics.prometheusRule.additionalLabels` | Additional labels that can be used so prometheusRules will be discovered by Prometheus | `{}` | +| `metrics.prometheusRule.namespace` | namespace where prometheusRules resource should be created | the same namespace as postgresql | +| `metrics.prometheusRule.rules` | [rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) to be created, check values for an example. | `[]` | +| `metrics.image.registry` | PostgreSQL Exporter Image registry | `docker.io` | +| `metrics.image.repository` | PostgreSQL Exporter Image name | `bitnami/postgres-exporter` | +| `metrics.image.tag` | PostgreSQL Exporter Image tag | `{TAG_NAME}` | +| `metrics.image.pullPolicy` | PostgreSQL Exporter Image pull policy | `IfNotPresent` | +| `metrics.image.pullSecrets` | Specify Image pull secrets | `nil` (does not add image pull secrets to deployed pods) | +| `metrics.customMetrics` | Additional custom metrics | `nil` | +| `metrics.extraEnvVars` | Extra environment variables to add to exporter | `{}` (evaluated as a template) | +| `metrics.securityContext.*` | Other container security context to be included as-is in the container spec | `{}` | +| `metrics.securityContext.enabled` | Enable security context for metrics | `false` | +| `metrics.securityContext.runAsUser` | User ID for the container for metrics | `1001` | +| `metrics.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 30 | +| `metrics.livenessProbe.periodSeconds` | How often to perform the probe | 10 | +| `metrics.livenessProbe.timeoutSeconds` | When the probe times out | 5 | +| `metrics.livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `metrics.livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `metrics.readinessProbe.enabled` | would you like a readinessProbe to be enabled | `true` | +| `metrics.readinessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 5 | +| `metrics.readinessProbe.periodSeconds` | How often to perform the probe | 10 | +| `metrics.readinessProbe.timeoutSeconds` | When the probe times out | 5 | +| `metrics.readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | 6 | +| `metrics.readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed | 1 | +| `updateStrategy` | Update strategy policy | `{type: "RollingUpdate"}` | +| `psp.create` | Create Pod Security Policy | `false` | +| `rbac.create` | Create Role and RoleBinding (required for PSP to work) | `false` | +| `extraDeploy` | Array of extra objects to deploy with the release (evaluated as a template). | `nil` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install my-release \ + --set postgresqlPassword=secretpassword,postgresqlDatabase=my-database \ + bitnami/postgresql +``` + +The above command sets the PostgreSQL `postgres` account password to `secretpassword`. Additionally it creates a database named `my-database`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```console +$ helm install my-release -f values.yaml bitnami/postgresql +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Production configuration and horizontal scaling + +This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one. + +- Enable replication: +```diff +- replication.enabled: false ++ replication.enabled: true +``` + +- Number of slaves replicas: +```diff +- replication.slaveReplicas: 1 ++ replication.slaveReplicas: 2 +``` + +- Set synchronous commit mode: +```diff +- replication.synchronousCommit: "off" ++ replication.synchronousCommit: "on" +``` + +- Number of replicas that will have synchronous replication: +```diff +- replication.numSynchronousReplicas: 0 ++ replication.numSynchronousReplicas: 1 +``` + +- Start a prometheus exporter: +```diff +- metrics.enabled: false ++ metrics.enabled: true +``` + +To horizontally scale this chart, you can use the `--replicas` flag to modify the number of nodes in your PostgreSQL deployment. Also you can use the `values-production.yaml` file or modify the parameters shown above. + +### Customizing Master and Slave services in a replicated configuration + +At the top level, there is a service object which defines the services for both master and slave. For deeper customization, there are service objects for both the master and slave types individually. This allows you to override the values in the top level service object so that the master and slave can be of different service types and with different clusterIPs / nodePorts. Also in the case you want the master and slave to be of type nodePort, you will need to set the nodePorts to different values to prevent a collision. The values that are deeper in the master.service or slave.service objects will take precedence over the top level service object. + +### Change PostgreSQL version + +To modify the PostgreSQL version used in this chart you can specify a [valid image tag](https://hub.docker.com/r/bitnami/postgresql/tags/) using the `image.tag` parameter. For example, `image.tag=X.Y.Z`. This approach is also applicable to other images like exporters. + +### postgresql.conf / pg_hba.conf files as configMap + +This helm chart also supports to customize the whole configuration file. + +Add your custom file to "files/postgresql.conf" in your working directory. This file will be mounted as configMap to the containers and it will be used for configuring the PostgreSQL server. + +Alternatively, you can specify PostgreSQL configuration parameters using the `postgresqlConfiguration` parameter as a dict, using camelCase, e.g. {"sharedBuffers": "500MB"}. + +In addition to these options, you can also set an external ConfigMap with all the configuration files. This is done by setting the `configurationConfigMap` parameter. Note that this will override the two previous options. + +### Allow settings to be loaded from files other than the default `postgresql.conf` + +If you don't want to provide the whole PostgreSQL configuration file and only specify certain parameters, you can add your extended `.conf` files to "files/conf.d/" in your working directory. +Those files will be mounted as configMap to the containers adding/overwriting the default configuration using the `include_dir` directive that allows settings to be loaded from files other than the default `postgresql.conf`. + +Alternatively, you can also set an external ConfigMap with all the extra configuration files. This is done by setting the `extendedConfConfigMap` parameter. Note that this will override the previous option. + +### Initialize a fresh instance + +The [Bitnami PostgreSQL](https://github.com/bitnami/bitnami-docker-postgresql) image allows you to use your custom scripts to initialize a fresh instance. In order to execute the scripts, they must be located inside the chart folder `files/docker-entrypoint-initdb.d` so they can be consumed as a ConfigMap. + +Alternatively, you can specify custom scripts using the `initdbScripts` parameter as dict. + +In addition to these options, you can also set an external ConfigMap with all the initialization scripts. This is done by setting the `initdbScriptsConfigMap` parameter. Note that this will override the two previous options. If your initialization scripts contain sensitive information such as credentials or passwords, you can use the `initdbScriptsSecret` parameter. + +The allowed extensions are `.sh`, `.sql` and `.sql.gz`. + +### Securing traffic using TLS + +TLS support can be enabled in the chart by specifying the `tls.` parameters while creating a release. The following parameters should be configured to properly enable the TLS support in the chart: + +- `tls.enabled`: Enable TLS support. Defaults to `false` +- `tls.certificatesSecret`: Name of an existing secret that contains the certificates. No defaults. +- `tls.certFilename`: Certificate filename. No defaults. +- `tls.certKeyFilename`: Certificate key filename. No defaults. + +For example: + +* First, create the secret with the cetificates files: + + ```console + kubectl create secret generic certificates-tls-secret --from-file=./cert.crt --from-file=./cert.key --from-file=./ca.crt + ``` + +* Then, use the following parameters: + + ```console + volumePermissions.enabled=true + tls.enabled=true + tls.certificatesSecret="certificates-tls-secret" + tls.certFilename="cert.crt" + tls.certKeyFilename="cert.key" + ``` + + > Note TLS and VolumePermissions: PostgreSQL requires certain permissions on sensitive files (such as certificate keys) to start up. Due to an on-going [issue](https://github.com/kubernetes/kubernetes/issues/57923) regarding kubernetes permissions and the use of `containerSecurityContext.runAsUser`, you must enable `volumePermissions` to ensure everything works as expected. + +### Sidecars + +If you need additional containers to run within the same pod as PostgreSQL (e.g. an additional metrics or logging exporter), you can do so via the `sidecars` config parameter. Simply define your container according to the Kubernetes container spec. + +```yaml +# For the PostgreSQL master +master: + sidecars: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +# For the PostgreSQL replicas +slave: + sidecars: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +``` + +### Metrics + +The chart optionally can start a metrics exporter for [prometheus](https://prometheus.io). The metrics endpoint (port 9187) is not exposed and it is expected that the metrics are collected from inside the k8s cluster using something similar as the described in the [example Prometheus scrape configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml). + +The exporter allows to create custom metrics from additional SQL queries. See the Chart's `values.yaml` for an example and consult the [exporters documentation](https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file) for more details. + +### Use of global variables + +In more complex scenarios, we may have the following tree of dependencies + +``` + +--------------+ + | | + +------------+ Chart 1 +-----------+ + | | | | + | --------+------+ | + | | | + | | | + | | | + | | | + v v v ++-------+------+ +--------+------+ +--------+------+ +| | | | | | +| PostgreSQL | | Sub-chart 1 | | Sub-chart 2 | +| | | | | | ++--------------+ +---------------+ +---------------+ +``` + +The three charts below depend on the parent chart Chart 1. However, subcharts 1 and 2 may need to connect to PostgreSQL as well. In order to do so, subcharts 1 and 2 need to know the PostgreSQL credentials, so one option for deploying could be deploy Chart 1 with the following parameters: + +``` +postgresql.postgresqlPassword=testtest +subchart1.postgresql.postgresqlPassword=testtest +subchart2.postgresql.postgresqlPassword=testtest +postgresql.postgresqlDatabase=db1 +subchart1.postgresql.postgresqlDatabase=db1 +subchart2.postgresql.postgresqlDatabase=db1 +``` + +If the number of dependent sub-charts increases, installing the chart with parameters can become increasingly difficult. An alternative would be to set the credentials using global variables as follows: + +``` +global.postgresql.postgresqlPassword=testtest +global.postgresql.postgresqlDatabase=db1 +``` + +This way, the credentials will be available in all of the subcharts. + +## Persistence + +The [Bitnami PostgreSQL](https://github.com/bitnami/bitnami-docker-postgresql) image stores the PostgreSQL data and configurations at the `/bitnami/postgresql` path of the container. + +Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. +See the [Parameters](#parameters) section to configure the PVC or to disable persistence. + +If you already have data in it, you will fail to sync to standby nodes for all commits, details can refer to [code](https://github.com/bitnami/bitnami-docker-postgresql/blob/8725fe1d7d30ebe8d9a16e9175d05f7ad9260c93/9.6/debian-9/rootfs/libpostgresql.sh#L518-L556). If you need to use those data, please covert them to sql and import after `helm install` finished. + +## NetworkPolicy + +To enable network policy for PostgreSQL, install [a networking plugin that implements the Kubernetes NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), and set `networkPolicy.enabled` to `true`. + +For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting the DefaultDeny namespace annotation. Note: this will enforce policy for _all_ pods in the namespace: + +```console +$ kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" +``` + +With NetworkPolicy enabled, traffic will be limited to just port 5432. + +For more precise policy, set `networkPolicy.allowExternal=false`. This will only allow pods with the generated client label to connect to PostgreSQL. +This label will be displayed in the output of a successful install. + +## Differences between Bitnami PostgreSQL image and [Docker Official](https://hub.docker.com/_/postgres) image + +- The Docker Official PostgreSQL image does not support replication. If you pass any replication environment variable, this would be ignored. The only environment variables supported by the Docker Official image are POSTGRES_USER, POSTGRES_DB, POSTGRES_PASSWORD, POSTGRES_INITDB_ARGS, POSTGRES_INITDB_WALDIR and PGDATA. All the remaining environment variables are specific to the Bitnami PostgreSQL image. +- The Bitnami PostgreSQL image is non-root by default. This requires that you run the pod with `securityContext` and updates the permissions of the volume with an `initContainer`. A key benefit of this configuration is that the pod follows security best practices and is prepared to run on Kubernetes distributions with hard security constraints like OpenShift. +- For OpenShift, one may either define the runAsUser and fsGroup accordingly, or try this more dynamic option: volumePermissions.securityContext.runAsUser="auto",securityContext.enabled=false,containerSecurityContext.enabled=false,shmVolume.chmod.enabled=false + +### Deploy chart using Docker Official PostgreSQL Image + +From chart version 4.0.0, it is possible to use this chart with the Docker Official PostgreSQL image. +Besides specifying the new Docker repository and tag, it is important to modify the PostgreSQL data directory and volume mount point. Basically, the PostgreSQL data dir cannot be the mount point directly, it has to be a subdirectory. + +``` +image.repository=postgres +image.tag=10.6 +postgresqlDataDir=/data/pgdata +persistence.mountPath=/data/ +``` + +## Upgrade + +It's necessary to specify the existing passwords while performing an upgrade to ensure the secrets are not updated with invalid randomly generated passwords. Remember to specify the existing values of the `postgresqlPassword` and `replication.password` parameters when upgrading the chart: + +```bash +$ helm upgrade my-release stable/postgresql \ + --set postgresqlPassword=[POSTGRESQL_PASSWORD] \ + --set replication.password=[REPLICATION_PASSWORD] +``` + +> Note: you need to substitute the placeholders _[POSTGRESQL_PASSWORD]_, and _[REPLICATION_PASSWORD]_ with the values obtained from instructions in the installation notes. + +## 9.0.0 + +In this version the chart was adapted to follow the Helm label best practices, see [PR 3021](https://github.com/bitnami/charts/pull/3021). That means the backward compatibility is not guarantee when upgrading the chart to this major version. + +As a workaround, you can delete the existing statefulset (using the `--cascade=false` flag pods are not deleted) before upgrade the chart. For example, this can be a valid workflow: + +- Deploy an old version (8.X.X) +```console +$ helm install postgresql bitnami/postgresql --version 8.10.14 +``` + +- Old version is up and running +```console +$ helm ls +NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION +postgresql default 1 2020-08-04 13:39:54.783480286 +0000 UTC deployed postgresql-8.10.14 11.8.0 + +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +postgresql-postgresql-0 1/1 Running 0 76s +``` + +- The upgrade to the latest one (9.X.X) is going to fail +```console +$ helm upgrade postgresql bitnami/postgresql +Error: UPGRADE FAILED: cannot patch "postgresql-postgresql" with kind StatefulSet: StatefulSet.apps "postgresql-postgresql" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', 'template', and 'updateStrategy' are forbidden +``` + +- Delete the statefulset +```console +$ kubectl delete statefulsets.apps --cascade=false postgresql-postgresql +statefulset.apps "postgresql-postgresql" deleted +``` + +- Now the upgrade works +```cosnole +$ helm upgrade postgresql bitnami/postgresql +$ helm ls +NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION +postgresql default 3 2020-08-04 13:42:08.020385884 +0000 UTC deployed postgresql-9.1.2 11.8.0 +``` + +- We can kill the existing pod and the new statefulset is going to create a new one: +```console +$ kubectl delete pod postgresql-postgresql-0 +pod "postgresql-postgresql-0" deleted + +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +postgresql-postgresql-0 1/1 Running 0 19s +``` + +Please, note that without the `--cascade=false` both objects (statefulset and pod) are going to be removed and both objects will be deployed again with the `helm upgrade` command + +## 8.0.0 + +Prefixes the port names with their protocols to comply with Istio conventions. + +If you depend on the port names in your setup, make sure to update them to reflect this change. + +## 7.1.0 + +Adds support for LDAP configuration. + +## 7.0.0 + +Helm performs a lookup for the object based on its group (apps), version (v1), and kind (Deployment). Also known as its GroupVersionKind, or GVK. Changing the GVK is considered a compatibility breaker from Kubernetes' point of view, so you cannot "upgrade" those objects to the new GVK in-place. Earlier versions of Helm 3 did not perform the lookup correctly which has since been fixed to match the spec. + +In https://github.com/helm/charts/pull/17281 the `apiVersion` of the statefulset resources was updated to `apps/v1` in tune with the api's deprecated, resulting in compatibility breakage. + +This major version bump signifies this change. + +## 6.5.7 + +In this version, the chart will use PostgreSQL with the Postgis extension included. The version used with Postgresql version 10, 11 and 12 is Postgis 2.5. It has been compiled with the following dependencies: + +- protobuf +- protobuf-c +- json-c +- geos +- proj + +## 5.0.0 + +In this version, the **chart is using PostgreSQL 11 instead of PostgreSQL 10**. You can find the main difference and notable changes in the following links: [https://www.postgresql.org/about/news/1894/](https://www.postgresql.org/about/news/1894/) and [https://www.postgresql.org/about/featurematrix/](https://www.postgresql.org/about/featurematrix/). + +For major releases of PostgreSQL, the internal data storage format is subject to change, thus complicating upgrades, you can see some errors like the following one in the logs: + +```console +Welcome to the Bitnami postgresql container +Subscribe to project updates by watching https://github.com/bitnami/bitnami-docker-postgresql +Submit issues and feature requests at https://github.com/bitnami/bitnami-docker-postgresql/issues +Send us your feedback at containers@bitnami.com + +INFO ==> ** Starting PostgreSQL setup ** +NFO ==> Validating settings in POSTGRESQL_* env vars.. +INFO ==> Initializing PostgreSQL database... +INFO ==> postgresql.conf file not detected. Generating it... +INFO ==> pg_hba.conf file not detected. Generating it... +INFO ==> Deploying PostgreSQL with persisted data... +INFO ==> Configuring replication parameters +INFO ==> Loading custom scripts... +INFO ==> Enabling remote connections +INFO ==> Stopping PostgreSQL... +INFO ==> ** PostgreSQL setup finished! ** + +INFO ==> ** Starting PostgreSQL ** + [1] FATAL: database files are incompatible with server + [1] DETAIL: The data directory was initialized by PostgreSQL version 10, which is not compatible with this version 11.3. +``` + +In this case, you should migrate the data from the old chart to the new one following an approach similar to that described in [this section](https://www.postgresql.org/docs/current/upgrading.html#UPGRADING-VIA-PGDUMPALL) from the official documentation. Basically, create a database dump in the old chart, move and restore it in the new one. + +### 4.0.0 + +This chart will use by default the Bitnami PostgreSQL container starting from version `10.7.0-r68`. This version moves the initialization logic from node.js to bash. This new version of the chart requires setting the `POSTGRES_PASSWORD` in the slaves as well, in order to properly configure the `pg_hba.conf` file. Users from previous versions of the chart are advised to upgrade immediately. + +IMPORTANT: If you do not want to upgrade the chart version then make sure you use the `10.7.0-r68` version of the container. Otherwise, you will get this error + +``` +The POSTGRESQL_PASSWORD environment variable is empty or not set. Set the environment variable ALLOW_EMPTY_PASSWORD=yes to allow the container to be started with blank passwords. This is recommended only for development +``` + +### 3.0.0 + +This releases make it possible to specify different nodeSelector, affinity and tolerations for master and slave pods. +It also fixes an issue with `postgresql.master.fullname` helper template not obeying fullnameOverride. + +#### Breaking changes + +- `affinty` has been renamed to `master.affinity` and `slave.affinity`. +- `tolerations` has been renamed to `master.tolerations` and `slave.tolerations`. +- `nodeSelector` has been renamed to `master.nodeSelector` and `slave.nodeSelector`. + +### 2.0.0 + +In order to upgrade from the `0.X.X` branch to `1.X.X`, you should follow the below steps: + + - Obtain the service name (`SERVICE_NAME`) and password (`OLD_PASSWORD`) of the existing postgresql chart. You can find the instructions to obtain the password in the NOTES.txt, the service name can be obtained by running + +```console +$ kubectl get svc +``` + +- Install (not upgrade) the new version + +```console +$ helm repo update +$ helm install my-release bitnami/postgresql +``` + +- Connect to the new pod (you can obtain the name by running `kubectl get pods`): + +```console +$ kubectl exec -it NAME bash +``` + +- Once logged in, create a dump file from the previous database using `pg_dump`, for that we should connect to the previous postgresql chart: + +```console +$ pg_dump -h SERVICE_NAME -U postgres DATABASE_NAME > /tmp/backup.sql +``` + +After run above command you should be prompted for a password, this password is the previous chart password (`OLD_PASSWORD`). +This operation could take some time depending on the database size. + +- Once you have the backup file, you can restore it with a command like the one below: + +```console +$ psql -U postgres DATABASE_NAME < /tmp/backup.sql +``` + +In this case, you are accessing to the local postgresql, so the password should be the new one (you can find it in NOTES.txt). + +If you want to restore the database and the database schema does not exist, it is necessary to first follow the steps described below. + +```console +$ psql -U postgres +postgres=# drop database DATABASE_NAME; +postgres=# create database DATABASE_NAME; +postgres=# create user USER_NAME; +postgres=# alter role USER_NAME with password 'BITNAMI_USER_PASSWORD'; +postgres=# grant all privileges on database DATABASE_NAME to USER_NAME; +postgres=# alter database DATABASE_NAME owner to USER_NAME; +``` diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/.helmignore b/scripts/helm/helm/databases/charts/postgresql/charts/common/.helmignore new file mode 100755 index 000000000..50af03172 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/Chart.yaml b/scripts/helm/helm/databases/charts/postgresql/charts/common/Chart.yaml new file mode 100755 index 000000000..5566cdc21 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + category: Infrastructure +apiVersion: v1 +appVersion: 0.8.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: common +sources: +- https://github.com/bitnami/charts +- http://www.bitnami.com/ +version: 0.8.1 diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/README.md b/scripts/helm/helm/databases/charts/postgresql/charts/common/README.md new file mode 100755 index 000000000..acdbe7bfa --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/README.md @@ -0,0 +1,286 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 0.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.12+ or Helm 3.0-beta3+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------------|-----------------------------------------------------------------|------------------------------------------------------------------| +| `common.affinities.node.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.node.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pod.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pod.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|-----------------------------------------------------------------|----------------------------| +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | + +### Labels + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|-----------------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Return the proper Docker Image Registry Secret Names | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|-----------------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | + +### Storage + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.affinities.node.soft` | Return a soft nodeAffinity definition | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frecuently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "context" $` secret and field are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|--------------------------------|-----------------------------------------------------------------|------------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets. + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 --decode) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 --decode) +``` + +## Notable changes + +N/A diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_affinities.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_affinities.tpl new file mode 100755 index 000000000..40f575cb6 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_affinities.tpl @@ -0,0 +1,94 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + namespaces: + - {{ .context.Release.Namespace }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + namespaces: + - {{ .context.Release.Namespace }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_capabilities.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_capabilities.tpl new file mode 100755 index 000000000..143bef2a4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_capabilities.tpl @@ -0,0 +1,33 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_errors.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_errors.tpl new file mode 100755 index 000000000..d6d3ec65a --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_errors.tpl @@ -0,0 +1,20 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: you must provide your current passwords when upgrade the release%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_images.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_images.tpl new file mode 100755 index 000000000..aafde9f3b --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_images.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_labels.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_labels.tpl new file mode 100755 index 000000000..252066c7e --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_names.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_names.tpl new file mode 100755 index 000000000..adf2a74f4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_names.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_secrets.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_secrets.tpl new file mode 100755 index 000000000..8eee91d21 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_secrets.tpl @@ -0,0 +1,49 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret - Optional. The path to the existing secrets in the values.yaml given by the user + to be used istead of the default one. +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- $name = .name -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret - Optional. The path to the existing secrets in the values.yaml given by the user + to be used istead of the default one. +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_storage.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_storage.tpl new file mode 100755 index 000000000..60e2a844f --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl new file mode 100755 index 000000000..2db166851 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_utils.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_utils.tpl new file mode 100755 index 000000000..74774a3ca --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_utils.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 --decode) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_validations.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_validations.tpl new file mode 100755 index 000000000..05d1edbaf --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_validations.tpl @@ -0,0 +1,278 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s=$%s' to the command.%s" .valueKey .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} + +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + + {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliar function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_warnings.tpl b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_warnings.tpl new file mode 100755 index 000000000..ae10fa41e --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/values.yaml b/scripts/helm/helm/databases/charts/postgresql/charts/common/values.yaml new file mode 100755 index 000000000..9ecdc93f5 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/charts/common/values.yaml @@ -0,0 +1,3 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +exampleValue: common-chart diff --git a/scripts/helm/helm/databases/charts/postgresql/ci/commonAnnotations.yaml b/scripts/helm/helm/databases/charts/postgresql/ci/commonAnnotations.yaml new file mode 100755 index 000000000..f6977823c --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/ci/commonAnnotations.yaml @@ -0,0 +1,3 @@ +commonAnnotations: + helm.sh/hook: 'pre-install, pre-upgrade' + helm.sh/hook-weight: '-1' diff --git a/scripts/helm/helm/databases/charts/postgresql/ci/default-values.yaml b/scripts/helm/helm/databases/charts/postgresql/ci/default-values.yaml new file mode 100755 index 000000000..fc2ba605a --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/ci/default-values.yaml @@ -0,0 +1 @@ +# Leave this file empty to ensure that CI runs builds against the default configuration in values.yaml. diff --git a/scripts/helm/helm/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml b/scripts/helm/helm/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml new file mode 100755 index 000000000..347d3b40a --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml @@ -0,0 +1,2 @@ +shmVolume: + enabled: false diff --git a/scripts/helm/helm/databases/charts/postgresql/files/README.md b/scripts/helm/helm/databases/charts/postgresql/files/README.md new file mode 100755 index 000000000..1813a2fea --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/files/README.md @@ -0,0 +1 @@ +Copy here your postgresql.conf and/or pg_hba.conf files to use it as a config map. diff --git a/scripts/helm/helm/databases/charts/postgresql/files/conf.d/README.md b/scripts/helm/helm/databases/charts/postgresql/files/conf.d/README.md new file mode 100755 index 000000000..184c1875d --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/files/conf.d/README.md @@ -0,0 +1,4 @@ +If you don't want to provide the whole configuration file and only specify certain parameters, you can copy here your extended `.conf` files. +These files will be injected as a config maps and add/overwrite the default configuration using the `include_dir` directive that allows settings to be loaded from files other than the default `postgresql.conf`. + +More info in the [bitnami-docker-postgresql README](https://github.com/bitnami/bitnami-docker-postgresql#configuration-file). diff --git a/scripts/helm/helm/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md b/scripts/helm/helm/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md new file mode 100755 index 000000000..cba38091e --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md @@ -0,0 +1,3 @@ +You can copy here your custom `.sh`, `.sql` or `.sql.gz` file so they are executed during the first boot of the image. + +More info in the [bitnami-docker-postgresql](https://github.com/bitnami/bitnami-docker-postgresql#initializing-a-new-instance) repository. \ No newline at end of file diff --git a/scripts/helm/helm/databases/charts/postgresql/postgresql.yaml b/scripts/helm/helm/databases/charts/postgresql/postgresql.yaml new file mode 100644 index 000000000..4e5e192d7 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/postgresql.yaml @@ -0,0 +1,185 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: db +--- +# Source: postgresql/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: postgres-postgresql + labels: + app.kubernetes.io/name: postgresql + helm.sh/chart: postgresql-9.8.2 + app.kubernetes.io/instance: postgres + app.kubernetes.io/managed-by: Helm +type: Opaque +data: + postgresql-password: "YXNheWVyUG9zdGdyZXM=" +--- +# Source: postgresql/templates/svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: postgres-postgresql-headless + labels: + app.kubernetes.io/name: postgresql + helm.sh/chart: postgresql-9.8.2 + app.kubernetes.io/instance: postgres + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-postgresql + port: 5432 + targetPort: tcp-postgresql + selector: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: postgres +--- +# Source: postgresql/templates/svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: postgres-postgresql + labels: + app.kubernetes.io/name: postgresql + helm.sh/chart: postgresql-9.8.2 + app.kubernetes.io/instance: postgres + app.kubernetes.io/managed-by: Helm + annotations: +spec: + type: ClusterIP + ports: + - name: tcp-postgresql + port: 5432 + targetPort: tcp-postgresql + selector: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: postgres + role: master +--- +# Source: postgresql/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres-postgresql + labels: + app.kubernetes.io/name: postgresql + helm.sh/chart: postgresql-9.8.2 + app.kubernetes.io/instance: postgres + app.kubernetes.io/managed-by: Helm + annotations: +spec: + serviceName: postgres-postgresql-headless + replicas: 1 + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: postgres + role: master + template: + metadata: + name: postgres-postgresql + labels: + app.kubernetes.io/name: postgresql + helm.sh/chart: postgresql-9.8.2 + app.kubernetes.io/instance: postgres + app.kubernetes.io/managed-by: Helm + role: master + spec: + securityContext: + fsGroup: 1001 + containers: + - name: postgres-postgresql + image: docker.io/bitnami/postgresql:11.9.0-debian-10-r48 + imagePullPolicy: "IfNotPresent" + resources: + requests: + cpu: 250m + memory: 256Mi + securityContext: + runAsUser: 1001 + env: + - name: BITNAMI_DEBUG + value: "false" + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: POSTGRESQL_VOLUME_DIR + value: "/bitnami/postgresql" + - name: PGDATA + value: "/bitnami/postgresql/data" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-postgresql + key: postgresql-password + - name: POSTGRESQL_ENABLE_LDAP + value: "no" + - name: POSTGRESQL_ENABLE_TLS + value: "no" + - name: POSTGRESQL_LOG_HOSTNAME + value: "false" + - name: POSTGRESQL_LOG_CONNECTIONS + value: "false" + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: "false" + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: "off" + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: "error" + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: "pgaudit" + ports: + - name: tcp-postgresql + containerPort: 5432 + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - -e + - | + exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 + [ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 6 + volumeMounts: + - name: dshm + mountPath: /dev/shm + - name: data + mountPath: /bitnami/postgresql + subPath: + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 1Gi + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" diff --git a/scripts/helm/helm/databases/charts/postgresql/requirements.lock b/scripts/helm/helm/databases/charts/postgresql/requirements.lock new file mode 100755 index 000000000..b0b7b0673 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 0.8.1 +digest: sha256:740783295d301fdd168fafdbaa760de27ab54b0ff36b513589a5a2515072b885 +generated: "2020-10-06T07:04:23.948475694Z" diff --git a/scripts/helm/helm/databases/charts/postgresql/requirements.yaml b/scripts/helm/helm/databases/charts/postgresql/requirements.yaml new file mode 100755 index 000000000..2c28bfe14 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: + - name: common + version: 0.x.x + repository: https://charts.bitnami.com/bitnami diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/NOTES.txt b/scripts/helm/helm/databases/charts/postgresql/templates/NOTES.txt new file mode 100755 index 000000000..596e969ce --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/NOTES.txt @@ -0,0 +1,59 @@ +** Please be patient while the chart is being deployed ** + +PostgreSQL can be accessed via port {{ template "postgresql.port" . }} on the following DNS name from within your cluster: + + {{ template "postgresql.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local - Read/Write connection +{{- if .Values.replication.enabled }} + {{ template "postgresql.fullname" . }}-read.{{ .Release.Namespace }}.svc.cluster.local - Read only connection +{{- end }} + +{{- if and (not (eq .Values.postgresqlUsername "postgres")) (or .Values.postgresqlPostgresPassword (include "postgresql.useExistingSecret" .)) }} + +To get the password for "postgres" run: + + export POSTGRES_ADMIN_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "postgresql.secretName" . }} -o jsonpath="{.data.postgresql-postgres-password}" | base64 --decode) +{{- end }} + +To get the password for "{{ template "postgresql.username" . }}" run: + + export POSTGRES_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "postgresql.secretName" . }} -o jsonpath="{.data.postgresql-password}" | base64 --decode) + +To connect to your database run the following command: + + kubectl run {{ template "postgresql.fullname" . }}-client --rm --tty -i --restart='Never' --namespace {{ .Release.Namespace }} --image {{ template "postgresql.image" . }} --env="PGPASSWORD=$POSTGRES_PASSWORD" {{- if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} + --labels="{{ template "postgresql.fullname" . }}-client=true" {{- end }} --command -- psql --host {{ template "postgresql.fullname" . }} -U {{ .Values.postgresqlUsername }} -d {{- if .Values.postgresqlDatabase }} {{ .Values.postgresqlDatabase }}{{- else }} postgres{{- end }} -p {{ template "postgresql.port" . }} + +{{ if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} +Note: Since NetworkPolicy is enabled, only pods with label {{ template "postgresql.fullname" . }}-client=true" will be able to connect to this PostgreSQL cluster. +{{- end }} + +To connect to your database from outside the cluster execute the following commands: + +{{- if contains "NodePort" .Values.service.type }} + + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "postgresql.fullname" . }}) + {{ if (include "postgresql.password" . ) }}PGPASSWORD="$POSTGRES_PASSWORD" {{ end }}psql --host $NODE_IP --port $NODE_PORT -U {{ .Values.postgresqlUsername }} -d {{- if .Values.postgresqlDatabase }} {{ .Values.postgresqlDatabase }}{{- else }} postgres{{- end }} + +{{- else if contains "LoadBalancer" .Values.service.type }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "postgresql.fullname" . }}' + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "postgresql.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + {{ if (include "postgresql.password" . ) }}PGPASSWORD="$POSTGRES_PASSWORD" {{ end }}psql --host $SERVICE_IP --port {{ template "postgresql.port" . }} -U {{ .Values.postgresqlUsername }} -d {{- if .Values.postgresqlDatabase }} {{ .Values.postgresqlDatabase }}{{- else }} postgres{{- end }} + +{{- else if contains "ClusterIP" .Values.service.type }} + + kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ template "postgresql.fullname" . }} {{ template "postgresql.port" . }}:{{ template "postgresql.port" . }} & + {{ if (include "postgresql.password" . ) }}PGPASSWORD="$POSTGRES_PASSWORD" {{ end }}psql --host 127.0.0.1 -U {{ .Values.postgresqlUsername }} -d {{- if .Values.postgresqlDatabase }} {{ .Values.postgresqlDatabase }}{{- else }} postgres{{- end }} -p {{ template "postgresql.port" . }} + +{{- end }} + +{{- include "postgresql.validateValues" . -}} + +{{- include "common.warnings.rollingTag" .Values.image -}} + +{{- $passwordValidationErrors := include "common.validations.values.postgresql.passwords" (dict "secret" (include "postgresql.fullname" .) "context" $) -}} + +{{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $passwordValidationErrors) "context" $) -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/_helpers.tpl b/scripts/helm/helm/databases/charts/postgresql/templates/_helpers.tpl new file mode 100755 index 000000000..b6a683ae9 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/_helpers.tpl @@ -0,0 +1,488 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "postgresql.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "postgresql.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "postgresql.master.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- $fullname := default (printf "%s-%s" .Release.Name $name) .Values.fullnameOverride -}} +{{- if .Values.replication.enabled -}} +{{- printf "%s-%s" $fullname "master" | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s" $fullname | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "postgresql.networkPolicy.apiVersion" -}} +{{- if semverCompare ">=1.4-0, <1.7-0" .Capabilities.KubeVersion.GitVersion -}} +"extensions/v1beta1" +{{- else if semverCompare "^1.7-0" .Capabilities.KubeVersion.GitVersion -}} +"networking.k8s.io/v1" +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "postgresql.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return the proper PostgreSQL image name +*/}} +{{- define "postgresql.image" -}} +{{- $registryName := .Values.image.registry -}} +{{- $repositoryName := .Values.image.repository -}} +{{- $tag := .Values.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL postgres user password +*/}} +{{- define "postgresql.postgres.password" -}} +{{- if .Values.global.postgresql.postgresqlPostgresPassword }} + {{- .Values.global.postgresql.postgresqlPostgresPassword -}} +{{- else if .Values.postgresqlPostgresPassword -}} + {{- .Values.postgresqlPostgresPassword -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL password +*/}} +{{- define "postgresql.password" -}} +{{- if .Values.global.postgresql.postgresqlPassword }} + {{- .Values.global.postgresql.postgresqlPassword -}} +{{- else if .Values.postgresqlPassword -}} + {{- .Values.postgresqlPassword -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL replication password +*/}} +{{- define "postgresql.replication.password" -}} +{{- if .Values.global.postgresql.replicationPassword }} + {{- .Values.global.postgresql.replicationPassword -}} +{{- else if .Values.replication.password -}} + {{- .Values.replication.password -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL username +*/}} +{{- define "postgresql.username" -}} +{{- if .Values.global.postgresql.postgresqlUsername }} + {{- .Values.global.postgresql.postgresqlUsername -}} +{{- else -}} + {{- .Values.postgresqlUsername -}} +{{- end -}} +{{- end -}} + + +{{/* +Return PostgreSQL replication username +*/}} +{{- define "postgresql.replication.username" -}} +{{- if .Values.global.postgresql.replicationUser }} + {{- .Values.global.postgresql.replicationUser -}} +{{- else -}} + {{- .Values.replication.user -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL port +*/}} +{{- define "postgresql.port" -}} +{{- if .Values.global.postgresql.servicePort }} + {{- .Values.global.postgresql.servicePort -}} +{{- else -}} + {{- .Values.service.port -}} +{{- end -}} +{{- end -}} + +{{/* +Return PostgreSQL created database +*/}} +{{- define "postgresql.database" -}} +{{- if .Values.global.postgresql.postgresqlDatabase }} + {{- .Values.global.postgresql.postgresqlDatabase -}} +{{- else if .Values.postgresqlDatabase -}} + {{- .Values.postgresqlDatabase -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper image name to change the volume permissions +*/}} +{{- define "postgresql.volumePermissions.image" -}} +{{- $registryName := .Values.volumePermissions.image.registry -}} +{{- $repositoryName := .Values.volumePermissions.image.repository -}} +{{- $tag := .Values.volumePermissions.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper PostgreSQL metrics image name +*/}} +{{- define "postgresql.metrics.image" -}} +{{- $registryName := default "docker.io" .Values.metrics.image.registry -}} +{{- $repositoryName := .Values.metrics.image.repository -}} +{{- $tag := default "latest" .Values.metrics.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Get the password secret. +*/}} +{{- define "postgresql.secretName" -}} +{{- if .Values.global.postgresql.existingSecret }} + {{- printf "%s" (tpl .Values.global.postgresql.existingSecret $) -}} +{{- else if .Values.existingSecret -}} + {{- printf "%s" (tpl .Values.existingSecret $) -}} +{{- else -}} + {{- printf "%s" (include "postgresql.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if we should use an existingSecret. +*/}} +{{- define "postgresql.useExistingSecret" -}} +{{- if or .Values.global.postgresql.existingSecret .Values.existingSecret -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a secret object should be created +*/}} +{{- define "postgresql.createSecret" -}} +{{- if not (include "postgresql.useExistingSecret" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Get the configuration ConfigMap name. +*/}} +{{- define "postgresql.configurationCM" -}} +{{- if .Values.configurationConfigMap -}} +{{- printf "%s" (tpl .Values.configurationConfigMap $) -}} +{{- else -}} +{{- printf "%s-configuration" (include "postgresql.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the extended configuration ConfigMap name. +*/}} +{{- define "postgresql.extendedConfigurationCM" -}} +{{- if .Values.extendedConfConfigMap -}} +{{- printf "%s" (tpl .Values.extendedConfConfigMap $) -}} +{{- else -}} +{{- printf "%s-extended-configuration" (include "postgresql.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if a configmap should be mounted with PostgreSQL configuration +*/}} +{{- define "postgresql.mountConfigurationCM" -}} +{{- if or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration .Values.configurationConfigMap }} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Get the initialization scripts ConfigMap name. +*/}} +{{- define "postgresql.initdbScriptsCM" -}} +{{- if .Values.initdbScriptsConfigMap -}} +{{- printf "%s" (tpl .Values.initdbScriptsConfigMap $) -}} +{{- else -}} +{{- printf "%s-init-scripts" (include "postgresql.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the initialization scripts Secret name. +*/}} +{{- define "postgresql.initdbScriptsSecret" -}} +{{- printf "%s" (tpl .Values.initdbScriptsSecret $) -}} +{{- end -}} + +{{/* +Get the metrics ConfigMap name. +*/}} +{{- define "postgresql.metricsCM" -}} +{{- printf "%s-metrics" (include "postgresql.fullname" .) -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "postgresql.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if or .Values.image.pullSecrets .Values.metrics.image.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.metrics.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- else if or .Values.image.pullSecrets .Values.metrics.image.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.metrics.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* +Get the readiness probe command +*/}} +{{- define "postgresql.readinessProbeCommand" -}} +- | +{{- if (include "postgresql.database" .) }} + exec pg_isready -U {{ include "postgresql.username" . | quote }} -d "dbname={{ include "postgresql.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.tlsCert" . }} sslkey={{ include "postgresql.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ template "postgresql.port" . }} +{{- else }} + exec pg_isready -U {{ include "postgresql.username" . | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.tlsCert" . }} sslkey={{ include "postgresql.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ template "postgresql.port" . }} +{{- end }} +{{- if contains "bitnami/" .Values.image.repository }} + [ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ] +{{- end -}} +{{- end -}} + +{{/* +Return the proper Storage Class +*/}} +{{- define "postgresql.storageClass" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +*/}} +{{- if .Values.global -}} + {{- if .Values.global.storageClass -}} + {{- if (eq "-" .Values.global.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.global.storageClass -}} + {{- end -}} + {{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.persistence.storageClass -}} + {{- if (eq "-" .Values.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.persistence.storageClass -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "postgresql.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "apps/v1beta2" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Compile all warnings into a single message, and call fail. +*/}} +{{- define "postgresql.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "postgresql.validateValues.ldapConfigurationMethod" .) -}} +{{- $messages := append $messages (include "postgresql.validateValues.psp" .) -}} +{{- $messages := append $messages (include "postgresql.validateValues.tls" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}} +{{- end -}} +{{- end -}} + +{{/* +Validate values of Postgresql - If ldap.url is used then you don't need the other settings for ldap +*/}} +{{- define "postgresql.validateValues.ldapConfigurationMethod" -}} +{{- if and .Values.ldap.enabled (and (not (empty .Values.ldap.url)) (not (empty .Values.ldap.server))) }} +postgresql: ldap.url, ldap.server + You cannot set both `ldap.url` and `ldap.server` at the same time. + Please provide a unique way to configure LDAP. + More info at https://www.postgresql.org/docs/current/auth-ldap.html +{{- end -}} +{{- end -}} + +{{/* +Validate values of Postgresql - If PSP is enabled RBAC should be enabled too +*/}} +{{- define "postgresql.validateValues.psp" -}} +{{- if and .Values.psp.create (not .Values.rbac.create) }} +postgresql: psp.create, rbac.create + RBAC should be enabled if PSP is enabled in order for PSP to work. + More info at https://kubernetes.io/docs/concepts/policy/pod-security-policy/#authorizing-policies +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for podsecuritypolicy. +*/}} +{{- define "podsecuritypolicy.apiVersion" -}} +{{- if semverCompare "<1.10-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Validate values of Postgresql TLS - When TLS is enabled, so must be VolumePermissions +*/}} +{{- define "postgresql.validateValues.tls" -}} +{{- if and .Values.tls.enabled (not .Values.volumePermissions.enabled) }} +postgresql: tls.enabled, volumePermissions.enabled + When TLS is enabled you must enable volumePermissions as well to ensure certificates files have + the right permissions. +{{- end -}} +{{- end -}} + +{{/* +Return the path to the cert file. +*/}} +{{- define "postgresql.tlsCert" -}} +{{- required "Certificate filename is required when TLS in enabled" .Values.tls.certFilename | printf "/opt/bitnami/postgresql/certs/%s" -}} +{{- end -}} + +{{/* +Return the path to the cert key file. +*/}} +{{- define "postgresql.tlsCertKey" -}} +{{- required "Certificate Key filename is required when TLS in enabled" .Values.tls.certKeyFilename | printf "/opt/bitnami/postgresql/certs/%s" -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "postgresql.tlsCACert" -}} +{{- printf "/opt/bitnami/postgresql/certs/%s" .Values.tls.certCAFilename -}} +{{- end -}} + +{{/* +Return the path to the CRL file. +*/}} +{{- define "postgresql.tlsCRL" -}} +{{- if .Values.tls.crlFilename -}} +{{- printf "/opt/bitnami/postgresql/certs/%s" .Values.tls.crlFilename -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/configmap.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/configmap.yaml new file mode 100755 index 000000000..bc78771d4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/configmap.yaml @@ -0,0 +1,26 @@ +{{ if and (or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration) (not .Values.configurationConfigMap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "postgresql.fullname" . }}-configuration + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: +{{- if (.Files.Glob "files/postgresql.conf") }} +{{ (.Files.Glob "files/postgresql.conf").AsConfig | indent 2 }} +{{- else if .Values.postgresqlConfiguration }} + postgresql.conf: | +{{- range $key, $value := default dict .Values.postgresqlConfiguration }} + {{ $key | snakecase }}={{ $value }} +{{- end }} +{{- end }} +{{- if (.Files.Glob "files/pg_hba.conf") }} +{{ (.Files.Glob "files/pg_hba.conf").AsConfig | indent 2 }} +{{- else if .Values.pgHbaConfiguration }} + pg_hba.conf: | +{{ .Values.pgHbaConfiguration | indent 4 }} +{{- end }} +{{ end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/extended-config-configmap.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/extended-config-configmap.yaml new file mode 100755 index 000000000..c6793802f --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/extended-config-configmap.yaml @@ -0,0 +1,21 @@ +{{- if and (or (.Files.Glob "files/conf.d/*.conf") .Values.postgresqlExtendedConf) (not .Values.extendedConfConfigMap)}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "postgresql.fullname" . }}-extended-configuration + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: +{{- with .Files.Glob "files/conf.d/*.conf" }} +{{ .AsConfig | indent 2 }} +{{- end }} +{{ with .Values.postgresqlExtendedConf }} + override.conf: | +{{- range $key, $value := . }} + {{ $key | snakecase }}={{ $value }} +{{- end }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/extra-list.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/extra-list.yaml new file mode 100755 index 000000000..b28a03c1c --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/extra-list.yaml @@ -0,0 +1,5 @@ +{{- if .Values.extraDeploy }} +apiVersion: v1 +kind: List +items: {{- include "postgresql.tplValue" (dict "value" .Values.extraDeploy "context" $) | nindent 2 }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/initialization-configmap.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/initialization-configmap.yaml new file mode 100755 index 000000000..2652ce732 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/initialization-configmap.yaml @@ -0,0 +1,24 @@ +{{- if and (or (.Files.Glob "files/docker-entrypoint-initdb.d/*.{sh,sql,sql.gz}") .Values.initdbScripts) (not .Values.initdbScriptsConfigMap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "postgresql.fullname" . }}-init-scripts + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +{{- with .Files.Glob "files/docker-entrypoint-initdb.d/*.sql.gz" }} +binaryData: +{{- range $path, $bytes := . }} + {{ base $path }}: {{ $.Files.Get $path | b64enc | quote }} +{{- end }} +{{- end }} +data: +{{- with .Files.Glob "files/docker-entrypoint-initdb.d/*.{sh,sql}" }} +{{ .AsConfig | indent 2 }} +{{- end }} +{{- with .Values.initdbScripts }} +{{ toYaml . | indent 2 }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/metrics-configmap.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/metrics-configmap.yaml new file mode 100755 index 000000000..6216eca84 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/metrics-configmap.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.metrics.enabled .Values.metrics.customMetrics }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "postgresql.metricsCM" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + custom-metrics.yaml: {{ toYaml .Values.metrics.customMetrics | quote }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/metrics-svc.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/metrics-svc.yaml new file mode 100755 index 000000000..9181ac89a --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/metrics-svc.yaml @@ -0,0 +1,25 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "postgresql.fullname" . }}-metrics + labels: + {{- include "common.labels.standard" . | nindent 4 }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- toYaml .Values.metrics.service.annotations | nindent 4 }} +spec: + type: {{ .Values.metrics.service.type }} + {{- if and (eq .Values.metrics.service.type "LoadBalancer") .Values.metrics.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.metrics.service.loadBalancerIP }} + {{- end }} + ports: + - name: http-metrics + port: 9187 + targetPort: http-metrics + selector: + {{- include "common.labels.matchLabels" . | nindent 4 }} + role: master +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/networkpolicy.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/networkpolicy.yaml new file mode 100755 index 000000000..f2752af77 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/networkpolicy.yaml @@ -0,0 +1,36 @@ +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: {{ template "postgresql.networkPolicy.apiVersion" . }} +metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} + ingress: + # Allow inbound connections + - ports: + - port: {{ template "postgresql.port" . }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "postgresql.fullname" . }}-client: "true" + {{- if .Values.networkPolicy.explicitNamespacesSelector }} + namespaceSelector: +{{ toYaml .Values.networkPolicy.explicitNamespacesSelector | indent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 14 }} + role: slave + {{- end }} + # Allow prometheus scrapes + - ports: + - port: 9187 +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/podsecuritypolicy.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..fb4c52f20 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/podsecuritypolicy.yaml @@ -0,0 +1,37 @@ +{{- if .Values.psp.create }} +apiVersion: {{ include "podsecuritypolicy.apiVersion" . }} +kind: PodSecurityPolicy +metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + privileged: false + volumes: + - 'configMap' + - 'secret' + - 'persistentVolumeClaim' + - 'emptyDir' + - 'projected' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/prometheusrule.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/prometheusrule.yaml new file mode 100755 index 000000000..0afd8f41f --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/prometheusrule.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ template "postgresql.fullname" . }} +{{- with .Values.metrics.prometheusRule.namespace }} + namespace: {{ . }} +{{- end }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- with .Values.metrics.prometheusRule.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: +{{- with .Values.metrics.prometheusRule.rules }} + groups: + - name: {{ template "postgresql.name" $ }} + rules: {{ tpl (toYaml .) $ | nindent 8 }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/role.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/role.yaml new file mode 100755 index 000000000..24148aa6b --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/role.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +rules: + {{- if .Values.psp.create }} + - apiGroups: ["extensions"] + resources: ["podsecuritypolicies"] + verbs: ["use"] + resourceNames: + - {{ template "postgresql.fullname" . }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/rolebinding.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/rolebinding.yaml new file mode 100755 index 000000000..a105fb41b --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +roleRef: + kind: Role + name: {{ template "postgresql.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ default (include "postgresql.fullname" . ) .Values.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/secrets.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/secrets.yaml new file mode 100755 index 000000000..8d968864c --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/secrets.yaml @@ -0,0 +1,23 @@ +{{- if (include "postgresql.createSecret" .) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + {{- if and .Values.postgresqlPostgresPassword (not (eq .Values.postgresqlUsername "postgres")) }} + postgresql-postgres-password: {{ include "postgresql.postgres.password" . | b64enc | quote }} + {{- end }} + postgresql-password: {{ include "postgresql.password" . | b64enc | quote }} + {{- if .Values.replication.enabled }} + postgresql-replication-password: {{ include "postgresql.replication.password" . | b64enc | quote }} + {{- end }} + {{- if (and .Values.ldap.enabled .Values.ldap.bind_password)}} + postgresql-ldap-password: {{ .Values.ldap.bind_password | b64enc | quote }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/serviceaccount.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/serviceaccount.yaml new file mode 100755 index 000000000..1e2a1f2a7 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if and (.Values.serviceAccount.enabled) (not .Values.serviceAccount.name) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "common.labels.standard" . | nindent 4 }} + name: {{ template "postgresql.fullname" . }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/servicemonitor.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/servicemonitor.yaml new file mode 100755 index 000000000..e118002a3 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "postgresql.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + +spec: + endpoints: + - port: http-metrics + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/statefulset-slaves.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/statefulset-slaves.yaml new file mode 100755 index 000000000..d77142fa3 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/statefulset-slaves.yaml @@ -0,0 +1,403 @@ +{{- if .Values.replication.enabled }} +{{- $slaveResources := coalesce .Values.slave.resources .Values.resources -}} +apiVersion: {{ template "postgresql.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: "{{ template "postgresql.fullname" . }}-slave" + labels: + {{- include "common.labels.standard" . | nindent 4 }} +{{- with .Values.slave.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- with .Values.slave.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + serviceName: {{ template "postgresql.fullname" . }}-headless + replicas: {{ .Values.replication.slaveReplicas }} + selector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} + role: slave + template: + metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 8 }} + role: slave +{{- with .Values.slave.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.slave.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: "{{ .Values.schedulerName }}" + {{- end }} +{{- include "postgresql.imagePullSecrets" . | indent 6 }} + {{- if .Values.slave.nodeSelector }} + nodeSelector: +{{ toYaml .Values.slave.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.slave.affinity }} + affinity: +{{ toYaml .Values.slave.affinity | indent 8 }} + {{- end }} + {{- if .Values.slave.tolerations }} + tolerations: +{{ toYaml .Values.slave.tolerations | indent 8 }} + {{- end }} + {{- if .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ default (include "postgresql.fullname" . ) .Values.serviceAccount.name}} + {{- end }} + {{- if or .Values.slave.extraInitContainers (and .Values.volumePermissions.enabled (or .Values.persistence.enabled (and .Values.shmVolume.enabled .Values.shmVolume.chmod.enabled))) }} + initContainers: + {{- if and .Values.volumePermissions.enabled (or .Values.persistence.enabled (and .Values.shmVolume.enabled .Values.shmVolume.chmod.enabled) .Values.tls.enabled) }} + - name: init-chmod-data + image: {{ template "postgresql.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -cx + - | + {{- if .Values.persistence.enabled }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown `id -u`:`id -G | cut -d " " -f2` {{ .Values.persistence.mountPath }} + {{- else }} + chown {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.persistence.mountPath }} + {{- end }} + mkdir -p {{ .Values.persistence.mountPath }}/data {{- if (include "postgresql.mountConfigurationCM" .) }} {{ .Values.persistence.mountPath }}/conf {{- end }} + chmod 700 {{ .Values.persistence.mountPath }}/data {{- if (include "postgresql.mountConfigurationCM" .) }} {{ .Values.persistence.mountPath }}/conf {{- end }} + find {{ .Values.persistence.mountPath }} -mindepth 1 -maxdepth 1 {{- if not (include "postgresql.mountConfigurationCM" .) }} -not -name "conf" {{- end }} -not -name ".snapshot" -not -name "lost+found" | \ + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + xargs chown -R `id -u`:`id -G | cut -d " " -f2` + {{- else }} + xargs chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} + {{- end }} + {{- end }} + {{- if and .Values.shmVolume.enabled .Values.shmVolume.chmod.enabled }} + chmod -R 777 /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + cp /tmp/certs/* /opt/bitnami/postgresql/certs/ + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` /opt/bitnami/postgresql/certs/ + {{- else }} + chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} /opt/bitnami/postgresql/certs/ + {{- end }} + chmod 600 {{ template "postgresql.tlsCertKey" . }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + securityContext: {{- omit .Values.volumePermissions.securityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.securityContext | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + {{ if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + mountPath: /tmp/certs + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + {{- end }} + {{- end }} + {{- if .Values.slave.extraInitContainers }} +{{ tpl .Values.slave.extraInitContainers . | indent 8 }} + {{- end }} + {{- end }} + {{- if .Values.slave.priorityClassName }} + priorityClassName: {{ .Values.slave.priorityClassName }} + {{- end }} + containers: + - name: {{ template "postgresql.fullname" . }} + image: {{ template "postgresql.image" . }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + {{- if $slaveResources }} + resources: {{- toYaml $slaveResources | nindent 12 }} + {{- end }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" .Values.image.debug | quote }} + - name: POSTGRESQL_VOLUME_DIR + value: "{{ .Values.persistence.mountPath }}" + - name: POSTGRESQL_PORT_NUMBER + value: "{{ template "postgresql.port" . }}" + {{- if .Values.persistence.mountPath }} + - name: PGDATA + value: {{ .Values.postgresqlDataDir | quote }} + {{- end }} + - name: POSTGRES_REPLICATION_MODE + value: "slave" + - name: POSTGRES_REPLICATION_USER + value: {{ include "postgresql.replication.username" . | quote }} + {{- if .Values.usePasswordFile }} + - name: POSTGRES_REPLICATION_PASSWORD_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-replication-password" + {{- else }} + - name: POSTGRES_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-replication-password + {{- end }} + - name: POSTGRES_CLUSTER_APP_NAME + value: {{ .Values.replication.applicationName }} + - name: POSTGRES_MASTER_HOST + value: {{ template "postgresql.fullname" . }} + - name: POSTGRES_MASTER_PORT_NUMBER + value: {{ include "postgresql.port" . | quote }} + {{- if and (not (eq .Values.postgresqlUsername "postgres")) (or .Values.postgresqlPostgresPassword (include "postgresql.useExistingSecret" .)) }} + {{- if .Values.usePasswordFile }} + - name: POSTGRES_POSTGRES_PASSWORD_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-postgres-password" + {{- else }} + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-postgres-password + {{- end }} + {{- end }} + {{- if .Values.usePasswordFile }} + - name: POSTGRES_PASSWORD_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-password" + {{- else }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-password + {{- end }} + - name: POSTGRESQL_ENABLE_TLS + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: POSTGRESQL_TLS_PREFER_SERVER_CIPHERS + value: {{ ternary "yes" "no" .Values.tls.preferServerCiphers | quote }} + - name: POSTGRESQL_TLS_CERT_FILE + value: {{ template "postgresql.tlsCert" . }} + - name: POSTGRESQL_TLS_KEY_FILE + value: {{ template "postgresql.tlsCertKey" . }} + {{- if .Values.tls.certCAFilename }} + - name: POSTGRESQL_TLS_CA_FILE + value: {{ template "postgresql.tlsCACert" . }} + {{- end }} + {{- if .Values.tls.crlFilename }} + - name: POSTGRESQL_TLS_CRL_FILE + value: {{ template "postgresql.tlsCRL" . }} + {{- end }} + {{- end }} + - name: POSTGRESQL_LOG_HOSTNAME + value: {{ .Values.audit.logHostname | quote }} + - name: POSTGRESQL_LOG_CONNECTIONS + value: {{ .Values.audit.logConnections | quote }} + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: {{ .Values.audit.logDisconnections | quote }} + {{- if .Values.audit.logLinePrefix }} + - name: POSTGRESQL_LOG_LINE_PREFIX + value: {{ .Values.audit.logLinePrefix | quote }} + {{- end }} + {{- if .Values.audit.logTimezone }} + - name: POSTGRESQL_LOG_TIMEZONE + value: {{ .Values.audit.logTimezone | quote }} + {{- end }} + {{- if .Values.audit.pgAuditLog }} + - name: POSTGRESQL_PGAUDIT_LOG + value: {{ .Values.audit.pgAuditLog | quote }} + {{- end }} + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: {{ .Values.audit.pgAuditLogCatalog | quote }} + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: {{ .Values.audit.clientMinMessages | quote }} + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: {{ .Values.postgresqlSharedPreloadLibraries | quote }} + {{- if .Values.postgresqlMaxConnections }} + - name: POSTGRESQL_MAX_CONNECTIONS + value: {{ .Values.postgresqlMaxConnections | quote }} + {{- end }} + {{- if .Values.postgresqlPostgresConnectionLimit }} + - name: POSTGRESQL_POSTGRES_CONNECTION_LIMIT + value: {{ .Values.postgresqlPostgresConnectionLimit | quote }} + {{- end }} + {{- if .Values.postgresqlDbUserConnectionLimit }} + - name: POSTGRESQL_USERNAME_CONNECTION_LIMIT + value: {{ .Values.postgresqlDbUserConnectionLimit | quote }} + {{- end }} + {{- if .Values.postgresqlTcpKeepalivesInterval }} + - name: POSTGRESQL_TCP_KEEPALIVES_INTERVAL + value: {{ .Values.postgresqlTcpKeepalivesInterval | quote }} + {{- end }} + {{- if .Values.postgresqlTcpKeepalivesIdle }} + - name: POSTGRESQL_TCP_KEEPALIVES_IDLE + value: {{ .Values.postgresqlTcpKeepalivesIdle | quote }} + {{- end }} + {{- if .Values.postgresqlStatementTimeout }} + - name: POSTGRESQL_STATEMENT_TIMEOUT + value: {{ .Values.postgresqlStatementTimeout | quote }} + {{- end }} + {{- if .Values.postgresqlTcpKeealivesCount }} + - name: POSTGRESQL_TCP_KEEPALIVES_COUNT + value: {{ .Values.postgresqlTcpKeealivesCount | quote }} + {{- end }} + {{- if .Values.postgresqlPghbaRemoveFilters }} + - name: POSTGRESQL_PGHBA_REMOVE_FILTERS + value: {{ .Values.postgresqlPghbaRemoveFilters | quote }} + {{- end }} + ports: + - name: tcp-postgresql + containerPort: {{ template "postgresql.port" . }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -c + {{- if (include "postgresql.database" .) }} + - exec pg_isready -U {{ include "postgresql.username" . | quote }} -d "dbname={{ include "postgresql.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.tlsCert" . }} sslkey={{ include "postgresql.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ template "postgresql.port" . }} + {{- else }} + - exec pg_isready -U {{ include "postgresql.username" . | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.tlsCert" . }} sslkey={{ include "postgresql.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ template "postgresql.port" . }} + {{- end }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- else if .Values.customLivenessProbe }} + livenessProbe: {{- include "postgresql.tplValue" (dict "value" .Values.customLivenessProbe "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - -e + {{- include "postgresql.readinessProbeCommand" . | nindent 16 }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- else if .Values.customReadinessProbe }} + readinessProbe: {{- include "postgresql.tplValue" (dict "value" .Values.customReadinessProbe "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.usePasswordFile }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + subPath: {{ .Values.persistence.subPath }} + {{ end }} + {{- if or (.Files.Glob "files/conf.d/*.conf") .Values.postgresqlExtendedConf .Values.extendedConfConfigMap }} + - name: postgresql-extended-config + mountPath: /bitnami/postgresql/conf/conf.d/ + {{- end }} + {{- if or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration .Values.configurationConfigMap }} + - name: postgresql-config + mountPath: /bitnami/postgresql/conf + {{- end }} + {{- if .Values.tls.enabled }} + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + readOnly: true + {{- end }} + {{- if .Values.slave.extraVolumeMounts }} + {{- toYaml .Values.slave.extraVolumeMounts | nindent 12 }} + {{- end }} +{{- if .Values.slave.sidecars }} +{{- include "common.tplvalues.render" ( dict "value" .Values.slave.sidecars "context" $ ) | nindent 8 }} +{{- end }} + volumes: + {{- if .Values.usePasswordFile }} + - name: postgresql-password + secret: + secretName: {{ template "postgresql.secretName" . }} + {{- end }} + {{- if or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration .Values.configurationConfigMap}} + - name: postgresql-config + configMap: + name: {{ template "postgresql.configurationCM" . }} + {{- end }} + {{- if or (.Files.Glob "files/conf.d/*.conf") .Values.postgresqlExtendedConf .Values.extendedConfConfigMap }} + - name: postgresql-extended-config + configMap: + name: {{ template "postgresql.extendedConfigurationCM" . }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + secret: + secretName: {{ required "A secret containing TLS certificates is required when TLS is enabled" .Values.tls.certificatesSecret }} + - name: postgresql-certificates + emptyDir: {} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 1Gi + {{- end }} + {{- if or (not .Values.persistence.enabled) (not .Values.slave.persistence.enabled) }} + - name: data + emptyDir: {} + {{- end }} + {{- if .Values.slave.extraVolumes }} + {{- toYaml .Values.slave.extraVolumes | nindent 8 }} + {{- end }} + updateStrategy: + type: {{ .Values.updateStrategy.type }} + {{- if (eq "Recreate" .Values.updateStrategy.type) }} + rollingUpdate: null + {{- end }} +{{- if and .Values.persistence.enabled .Values.slave.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + {{- with .Values.persistence.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{ include "postgresql.storageClass" . }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/statefulset.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/statefulset.yaml new file mode 100755 index 000000000..10c1af166 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/statefulset.yaml @@ -0,0 +1,580 @@ +apiVersion: {{ template "postgresql.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ template "postgresql.master.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- with .Values.master.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- with .Values.master.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + serviceName: {{ template "postgresql.fullname" . }}-headless + replicas: 1 + updateStrategy: + type: {{ .Values.updateStrategy.type }} + {{- if (eq "Recreate" .Values.updateStrategy.type) }} + rollingUpdate: null + {{- end }} + selector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} + role: master + template: + metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 8 }} + role: master + {{- with .Values.master.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.podAnnotations }} + annotations: {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: "{{ .Values.schedulerName }}" + {{- end }} +{{- include "postgresql.imagePullSecrets" . | indent 6 }} + {{- if .Values.master.nodeSelector }} + nodeSelector: {{- toYaml .Values.master.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.master.affinity }} + affinity: {{- toYaml .Values.master.affinity | nindent 8 }} + {{- end }} + {{- if .Values.master.tolerations }} + tolerations: {{- toYaml .Values.master.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.enabled }} + serviceAccountName: {{ default (include "postgresql.fullname" . ) .Values.serviceAccount.name }} + {{- end }} + {{- if or .Values.master.extraInitContainers (and .Values.volumePermissions.enabled (or .Values.persistence.enabled (and .Values.shmVolume.enabled .Values.shmVolume.chmod.enabled))) }} + initContainers: + {{- if and .Values.volumePermissions.enabled (or .Values.persistence.enabled (and .Values.shmVolume.enabled .Values.shmVolume.chmod.enabled) .Values.tls.enabled) }} + - name: init-chmod-data + image: {{ template "postgresql.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -cx + - | + {{- if .Values.persistence.enabled }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown `id -u`:`id -G | cut -d " " -f2` {{ .Values.persistence.mountPath }} + {{- else }} + chown {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.persistence.mountPath }} + {{- end }} + mkdir -p {{ .Values.persistence.mountPath }}/data {{- if (include "postgresql.mountConfigurationCM" .) }} {{ .Values.persistence.mountPath }}/conf {{- end }} + chmod 700 {{ .Values.persistence.mountPath }}/data {{- if (include "postgresql.mountConfigurationCM" .) }} {{ .Values.persistence.mountPath }}/conf {{- end }} + find {{ .Values.persistence.mountPath }} -mindepth 1 -maxdepth 1 {{- if not (include "postgresql.mountConfigurationCM" .) }} -not -name "conf" {{- end }} -not -name ".snapshot" -not -name "lost+found" | \ + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + xargs chown -R `id -u`:`id -G | cut -d " " -f2` + {{- else }} + xargs chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} + {{- end }} + {{- end }} + {{- if and .Values.shmVolume.enabled .Values.shmVolume.chmod.enabled }} + chmod -R 777 /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + cp /tmp/certs/* /opt/bitnami/postgresql/certs/ + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` /opt/bitnami/postgresql/certs/ + {{- else }} + chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} /opt/bitnami/postgresql/certs/ + {{- end }} + chmod 600 {{ template "postgresql.tlsCertKey" . }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + securityContext: {{- omit .Values.volumePermissions.securityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.securityContext | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + mountPath: /tmp/certs + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + {{- end }} + {{- end }} + {{- if .Values.master.extraInitContainers }} + {{- include "common.tplvalues.render" ( dict "value" .Values.master.extraInitContainers "context" $ ) | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.master.priorityClassName }} + priorityClassName: {{ .Values.master.priorityClassName }} + {{- end }} + containers: + - name: {{ template "postgresql.fullname" . }} + image: {{ template "postgresql.image" . }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" .Values.image.debug | quote }} + - name: POSTGRESQL_PORT_NUMBER + value: "{{ template "postgresql.port" . }}" + - name: POSTGRESQL_VOLUME_DIR + value: "{{ .Values.persistence.mountPath }}" + {{- if .Values.postgresqlInitdbArgs }} + - name: POSTGRES_INITDB_ARGS + value: {{ .Values.postgresqlInitdbArgs | quote }} + {{- end }} + {{- if .Values.postgresqlInitdbWalDir }} + - name: POSTGRES_INITDB_WALDIR + value: {{ .Values.postgresqlInitdbWalDir | quote }} + {{- end }} + {{- if .Values.initdbUser }} + - name: POSTGRESQL_INITSCRIPTS_USERNAME + value: {{ .Values.initdbUser }} + {{- end }} + {{- if .Values.initdbPassword }} + - name: POSTGRESQL_INITSCRIPTS_PASSWORD + value: {{ .Values.initdbPassword }} + {{- end }} + {{- if .Values.persistence.mountPath }} + - name: PGDATA + value: {{ .Values.postgresqlDataDir | quote }} + {{- end }} + {{- if .Values.masterAsStandBy.enabled }} + - name: POSTGRES_MASTER_HOST + value: {{ .Values.masterAsStandBy.masterHost }} + - name: POSTGRES_MASTER_PORT_NUMBER + value: {{ .Values.masterAsStandBy.masterPort | quote }} + {{- end }} + {{- if or .Values.replication.enabled .Values.masterAsStandBy.enabled }} + - name: POSTGRES_REPLICATION_MODE + {{- if .Values.masterAsStandBy.enabled }} + value: "slave" + {{- else }} + value: "master" + {{- end }} + - name: POSTGRES_REPLICATION_USER + value: {{ include "postgresql.replication.username" . | quote }} + {{- if .Values.usePasswordFile }} + - name: POSTGRES_REPLICATION_PASSWORD_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-replication-password" + {{- else }} + - name: POSTGRES_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-replication-password + {{- end }} + {{- if not (eq .Values.replication.synchronousCommit "off")}} + - name: POSTGRES_SYNCHRONOUS_COMMIT_MODE + value: {{ .Values.replication.synchronousCommit | quote }} + - name: POSTGRES_NUM_SYNCHRONOUS_REPLICAS + value: {{ .Values.replication.numSynchronousReplicas | quote }} + {{- end }} + - name: POSTGRES_CLUSTER_APP_NAME + value: {{ .Values.replication.applicationName }} + {{- end }} + {{- if and (not (eq .Values.postgresqlUsername "postgres")) (or .Values.postgresqlPostgresPassword (include "postgresql.useExistingSecret" .)) }} + {{- if .Values.usePasswordFile }} + - name: POSTGRES_POSTGRES_PASSWORD_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-postgres-password" + {{- else }} + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-postgres-password + {{- end }} + {{- end }} + - name: POSTGRES_USER + value: {{ include "postgresql.username" . | quote }} + {{- if .Values.usePasswordFile }} + - name: POSTGRES_PASSWORD_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-password" + {{- else }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-password + {{- end }} + {{- if (include "postgresql.database" .) }} + - name: POSTGRES_DB + value: {{ (include "postgresql.database" .) | quote }} + {{- end }} + {{- if .Values.extraEnv }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnv "context" $) | nindent 12 }} + {{- end }} + - name: POSTGRESQL_ENABLE_LDAP + value: {{ ternary "yes" "no" .Values.ldap.enabled | quote }} + {{- if .Values.ldap.enabled }} + - name: POSTGRESQL_LDAP_SERVER + value: {{ .Values.ldap.server }} + - name: POSTGRESQL_LDAP_PORT + value: {{ .Values.ldap.port | quote }} + - name: POSTGRESQL_LDAP_SCHEME + value: {{ .Values.ldap.scheme }} + {{- if .Values.ldap.tls }} + - name: POSTGRESQL_LDAP_TLS + value: "1" + {{- end }} + - name: POSTGRESQL_LDAP_PREFIX + value: {{ .Values.ldap.prefix | quote }} + - name: POSTGRESQL_LDAP_SUFFIX + value: {{ .Values.ldap.suffix | quote }} + - name: POSTGRESQL_LDAP_BASE_DN + value: {{ .Values.ldap.baseDN }} + - name: POSTGRESQL_LDAP_BIND_DN + value: {{ .Values.ldap.bindDN }} + {{- if (not (empty .Values.ldap.bind_password)) }} + - name: POSTGRESQL_LDAP_BIND_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-ldap-password + {{- end}} + - name: POSTGRESQL_LDAP_SEARCH_ATTR + value: {{ .Values.ldap.search_attr }} + - name: POSTGRESQL_LDAP_SEARCH_FILTER + value: {{ .Values.ldap.search_filter }} + - name: POSTGRESQL_LDAP_URL + value: {{ .Values.ldap.url }} + {{- end}} + - name: POSTGRESQL_ENABLE_TLS + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: POSTGRESQL_TLS_PREFER_SERVER_CIPHERS + value: {{ ternary "yes" "no" .Values.tls.preferServerCiphers | quote }} + - name: POSTGRESQL_TLS_CERT_FILE + value: {{ template "postgresql.tlsCert" . }} + - name: POSTGRESQL_TLS_KEY_FILE + value: {{ template "postgresql.tlsCertKey" . }} + {{- if .Values.tls.certCAFilename }} + - name: POSTGRESQL_TLS_CA_FILE + value: {{ template "postgresql.tlsCACert" . }} + {{- end }} + {{- if .Values.tls.crlFilename }} + - name: POSTGRESQL_TLS_CRL_FILE + value: {{ template "postgresql.tlsCRL" . }} + {{- end }} + {{- end }} + - name: POSTGRESQL_LOG_HOSTNAME + value: {{ .Values.audit.logHostname | quote }} + - name: POSTGRESQL_LOG_CONNECTIONS + value: {{ .Values.audit.logConnections | quote }} + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: {{ .Values.audit.logDisconnections | quote }} + {{- if .Values.audit.logLinePrefix }} + - name: POSTGRESQL_LOG_LINE_PREFIX + value: {{ .Values.audit.logLinePrefix | quote }} + {{- end }} + {{- if .Values.audit.logTimezone }} + - name: POSTGRESQL_LOG_TIMEZONE + value: {{ .Values.audit.logTimezone | quote }} + {{- end }} + {{- if .Values.audit.pgAuditLog }} + - name: POSTGRESQL_PGAUDIT_LOG + value: {{ .Values.audit.pgAuditLog | quote }} + {{- end }} + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: {{ .Values.audit.pgAuditLogCatalog | quote }} + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: {{ .Values.audit.clientMinMessages | quote }} + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: {{ .Values.postgresqlSharedPreloadLibraries | quote }} + {{- if .Values.postgresqlMaxConnections }} + - name: POSTGRESQL_MAX_CONNECTIONS + value: {{ .Values.postgresqlMaxConnections | quote }} + {{- end }} + {{- if .Values.postgresqlPostgresConnectionLimit }} + - name: POSTGRESQL_POSTGRES_CONNECTION_LIMIT + value: {{ .Values.postgresqlPostgresConnectionLimit | quote }} + {{- end }} + {{- if .Values.postgresqlDbUserConnectionLimit }} + - name: POSTGRESQL_USERNAME_CONNECTION_LIMIT + value: {{ .Values.postgresqlDbUserConnectionLimit | quote }} + {{- end }} + {{- if .Values.postgresqlTcpKeepalivesInterval }} + - name: POSTGRESQL_TCP_KEEPALIVES_INTERVAL + value: {{ .Values.postgresqlTcpKeepalivesInterval | quote }} + {{- end }} + {{- if .Values.postgresqlTcpKeepalivesIdle }} + - name: POSTGRESQL_TCP_KEEPALIVES_IDLE + value: {{ .Values.postgresqlTcpKeepalivesIdle | quote }} + {{- end }} + {{- if .Values.postgresqlStatementTimeout }} + - name: POSTGRESQL_STATEMENT_TIMEOUT + value: {{ .Values.postgresqlStatementTimeout | quote }} + {{- end }} + {{- if .Values.postgresqlTcpKeealivesCount }} + - name: POSTGRESQL_TCP_KEEPALIVES_COUNT + value: {{ .Values.postgresqlTcpKeealivesCount | quote }} + {{- end }} + {{- if .Values.postgresqlPghbaRemoveFilters }} + - name: POSTGRESQL_PGHBA_REMOVE_FILTERS + value: {{ .Values.postgresqlPghbaRemoveFilters | quote }} + {{- end }} + {{- if .Values.extraEnvVarsCM }} + envFrom: + - configMapRef: + name: {{ tpl .Values.extraEnvVarsCM . }} + {{- end }} + ports: + - name: tcp-postgresql + containerPort: {{ template "postgresql.port" . }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -c + {{- if (include "postgresql.database" .) }} + - exec pg_isready -U {{ include "postgresql.username" . | quote }} -d "dbname={{ include "postgresql.database" . }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} sslcert={{ include "postgresql.tlsCert" . }} sslkey={{ include "postgresql.tlsCertKey" . }}{{- end }}" -h 127.0.0.1 -p {{ template "postgresql.port" . }} + {{- else }} + - exec pg_isready -U {{ include "postgresql.username" . | quote }} {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} -d "sslcert={{ include "postgresql.tlsCert" . }} sslkey={{ include "postgresql.tlsCertKey" . }}"{{- end }} -h 127.0.0.1 -p {{ template "postgresql.port" . }} + {{- end }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- else if .Values.customLivenessProbe }} + livenessProbe: {{- include "postgresql.tplValue" (dict "value" .Values.customLivenessProbe "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -c + - -e + {{- include "postgresql.readinessProbeCommand" . | nindent 16 }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- else if .Values.customReadinessProbe }} + readinessProbe: {{- include "postgresql.tplValue" (dict "value" .Values.customReadinessProbe "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + {{- if or (.Files.Glob "files/docker-entrypoint-initdb.d/*.{sh,sql,sql.gz}") .Values.initdbScriptsConfigMap .Values.initdbScripts }} + - name: custom-init-scripts + mountPath: /docker-entrypoint-initdb.d/ + {{- end }} + {{- if .Values.initdbScriptsSecret }} + - name: custom-init-scripts-secret + mountPath: /docker-entrypoint-initdb.d/secret + {{- end }} + {{- if or (.Files.Glob "files/conf.d/*.conf") .Values.postgresqlExtendedConf .Values.extendedConfConfigMap }} + - name: postgresql-extended-config + mountPath: /bitnami/postgresql/conf/conf.d/ + {{- end }} + {{- if .Values.usePasswordFile }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + readOnly: true + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + mountPath: /dev/shm + {{- end }} + {{- if .Values.persistence.enabled }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + {{- if or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration .Values.configurationConfigMap }} + - name: postgresql-config + mountPath: /bitnami/postgresql/conf + {{- end }} + {{- if .Values.master.extraVolumeMounts }} + {{- toYaml .Values.master.extraVolumeMounts | nindent 12 }} + {{- end }} +{{- if .Values.master.sidecars }} +{{- include "common.tplvalues.render" ( dict "value" .Values.master.sidecars "context" $ ) | nindent 8 }} +{{- end }} +{{- if .Values.metrics.enabled }} + - name: metrics + image: {{ template "postgresql.metrics.image" . }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + {{- if .Values.metrics.securityContext.enabled }} + securityContext: {{- omit .Values.metrics.securityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + env: + {{- $database := required "In order to enable metrics you need to specify a database (.Values.postgresqlDatabase or .Values.global.postgresql.postgresqlDatabase)" (include "postgresql.database" .) }} + {{- $sslmode := ternary "require" "disable" .Values.tls.enabled }} + {{- if and .Values.tls.enabled .Values.tls.certCAFilename }} + - name: DATA_SOURCE_NAME + value: {{ printf "host=127.0.0.1 port=%d user=%s sslmode=%s sslcert=%s sslkey=%s" (int (include "postgresql.port" .)) (include "postgresql.username" .) $sslmode (include "postgresql.tlsCert" .) (include "postgresql.tlsCertKey" .) }} + {{- else }} + - name: DATA_SOURCE_URI + value: {{ printf "127.0.0.1:%d/%s?sslmode=%s" (int (include "postgresql.port" .)) $database $sslmode }} + {{- end }} + {{- if .Values.usePasswordFile }} + - name: DATA_SOURCE_PASS_FILE + value: "/opt/bitnami/postgresql/secrets/postgresql-password" + {{- else }} + - name: DATA_SOURCE_PASS + valueFrom: + secretKeyRef: + name: {{ template "postgresql.secretName" . }} + key: postgresql-password + {{- end }} + - name: DATA_SOURCE_USER + value: {{ template "postgresql.username" . }} + {{- if .Values.metrics.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.metrics.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: / + port: http-metrics + initialDelaySeconds: {{ .Values.metrics.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.metrics.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.metrics.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.metrics.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.metrics.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: / + port: http-metrics + initialDelaySeconds: {{ .Values.metrics.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.metrics.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.metrics.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.metrics.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.metrics.readinessProbe.failureThreshold }} + {{- end }} + volumeMounts: + {{- if .Values.usePasswordFile }} + - name: postgresql-password + mountPath: /opt/bitnami/postgresql/secrets/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: postgresql-certificates + mountPath: /opt/bitnami/postgresql/certs + readOnly: true + {{- end }} + {{- if .Values.metrics.customMetrics }} + - name: custom-metrics + mountPath: /conf + readOnly: true + args: ["--extend.query-path", "/conf/custom-metrics.yaml"] + {{- end }} + ports: + - name: http-metrics + containerPort: 9187 + {{- if .Values.metrics.resources }} + resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} +{{- end }} + volumes: + {{- if or (.Files.Glob "files/postgresql.conf") (.Files.Glob "files/pg_hba.conf") .Values.postgresqlConfiguration .Values.pgHbaConfiguration .Values.configurationConfigMap}} + - name: postgresql-config + configMap: + name: {{ template "postgresql.configurationCM" . }} + {{- end }} + {{- if or (.Files.Glob "files/conf.d/*.conf") .Values.postgresqlExtendedConf .Values.extendedConfConfigMap }} + - name: postgresql-extended-config + configMap: + name: {{ template "postgresql.extendedConfigurationCM" . }} + {{- end }} + {{- if .Values.usePasswordFile }} + - name: postgresql-password + secret: + secretName: {{ template "postgresql.secretName" . }} + {{- end }} + {{- if or (.Files.Glob "files/docker-entrypoint-initdb.d/*.{sh,sql,sql.gz}") .Values.initdbScriptsConfigMap .Values.initdbScripts }} + - name: custom-init-scripts + configMap: + name: {{ template "postgresql.initdbScriptsCM" . }} + {{- end }} + {{- if .Values.initdbScriptsSecret }} + - name: custom-init-scripts-secret + secret: + secretName: {{ template "postgresql.initdbScriptsSecret" . }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: raw-certificates + secret: + secretName: {{ required "A secret containing TLS certificates is required when TLS is enabled" .Values.tls.certificatesSecret }} + - name: postgresql-certificates + emptyDir: {} + {{- end }} + {{- if .Values.master.extraVolumes }} + {{- toYaml .Values.master.extraVolumes | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.customMetrics }} + - name: custom-metrics + configMap: + name: {{ template "postgresql.metricsCM" . }} + {{- end }} + {{- if .Values.shmVolume.enabled }} + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 1Gi + {{- end }} +{{- if and .Values.persistence.enabled .Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: +{{- with .Values.persistence.existingClaim }} + claimName: {{ tpl . $ }} +{{- end }} +{{- else if not .Values.persistence.enabled }} + - name: data + emptyDir: {} +{{- else if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: data + {{- with .Values.persistence.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{ include "postgresql.storageClass" . }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/svc-headless.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/svc-headless.yaml new file mode 100755 index 000000000..fb8c838d2 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/svc-headless.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "postgresql.fullname" . }}-headless + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-postgresql + port: {{ template "postgresql.port" . }} + targetPort: tcp-postgresql + selector: + {{- include "common.labels.matchLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/svc-read.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/svc-read.yaml new file mode 100755 index 000000000..5ee051c45 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/svc-read.yaml @@ -0,0 +1,42 @@ +{{- if .Values.replication.enabled }} +{{- $serviceAnnotations := coalesce .Values.slave.service.annotations .Values.service.annotations -}} +{{- $serviceType := coalesce .Values.slave.service.type .Values.service.type -}} +{{- $serviceLoadBalancerIP := coalesce .Values.slave.service.loadBalancerIP .Values.service.loadBalancerIP -}} +{{- $serviceLoadBalancerSourceRanges := coalesce .Values.slave.service.loadBalancerSourceRanges .Values.service.loadBalancerSourceRanges -}} +{{- $serviceClusterIP := coalesce .Values.slave.service.clusterIP .Values.service.clusterIP -}} +{{- $serviceNodePort := coalesce .Values.slave.service.nodePort .Values.service.nodePort -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "postgresql.fullname" . }}-read + labels: + {{- include "common.labels.standard" . | nindent 4 }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if $serviceAnnotations }} + {{- include "common.tplvalues.render" (dict "value" $serviceAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: {{ $serviceType }} + {{- if and $serviceLoadBalancerIP (eq $serviceType "LoadBalancer") }} + loadBalancerIP: {{ $serviceLoadBalancerIP }} + {{- end }} + {{- if and (eq $serviceType "LoadBalancer") $serviceLoadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- include "common.tplvalues.render" (dict "value" $serviceLoadBalancerSourceRanges "context" $) | nindent 4 }} + {{- end }} + {{- if and (eq $serviceType "ClusterIP") $serviceClusterIP }} + clusterIP: {{ $serviceClusterIP }} + {{- end }} + ports: + - name: tcp-postgresql + port: {{ template "postgresql.port" . }} + targetPort: tcp-postgresql + {{- if $serviceNodePort }} + nodePort: {{ $serviceNodePort }} + {{- end }} + selector: + {{- include "common.labels.matchLabels" . | nindent 4 }} + role: slave +{{- end }} diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/svc.yaml b/scripts/helm/helm/databases/charts/postgresql/templates/svc.yaml new file mode 100755 index 000000000..3dbfaa12d --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/templates/svc.yaml @@ -0,0 +1,40 @@ +{{- $serviceAnnotations := coalesce .Values.master.service.annotations .Values.service.annotations -}} +{{- $serviceType := coalesce .Values.master.service.type .Values.service.type -}} +{{- $serviceLoadBalancerIP := coalesce .Values.master.service.loadBalancerIP .Values.service.loadBalancerIP -}} +{{- $serviceLoadBalancerSourceRanges := coalesce .Values.master.service.loadBalancerSourceRanges .Values.service.loadBalancerSourceRanges -}} +{{- $serviceClusterIP := coalesce .Values.master.service.clusterIP .Values.service.clusterIP -}} +{{- $serviceNodePort := coalesce .Values.master.service.nodePort .Values.service.nodePort -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "postgresql.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + annotations: + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if $serviceAnnotations }} + {{- include "common.tplvalues.render" (dict "value" $serviceAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: {{ $serviceType }} + {{- if and $serviceLoadBalancerIP (eq $serviceType "LoadBalancer") }} + loadBalancerIP: {{ $serviceLoadBalancerIP }} + {{- end }} + {{- if and (eq $serviceType "LoadBalancer") $serviceLoadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- include "common.tplvalues.render" (dict "value" $serviceLoadBalancerSourceRanges "context" $) | nindent 4 }} + {{- end }} + {{- if and (eq $serviceType "ClusterIP") $serviceClusterIP }} + clusterIP: {{ $serviceClusterIP }} + {{- end }} + ports: + - name: tcp-postgresql + port: {{ template "postgresql.port" . }} + targetPort: tcp-postgresql + {{- if $serviceNodePort }} + nodePort: {{ $serviceNodePort }} + {{- end }} + selector: + {{- include "common.labels.matchLabels" . | nindent 4 }} + role: master diff --git a/scripts/helm/helm/databases/charts/postgresql/values-production.yaml b/scripts/helm/helm/databases/charts/postgresql/values-production.yaml new file mode 100755 index 000000000..3e144c131 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/values-production.yaml @@ -0,0 +1,711 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + postgresql: {} +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami PostgreSQL image version +## ref: https://hub.docker.com/r/bitnami/postgresql/tags/ +## +image: + registry: docker.io + repository: bitnami/postgresql + tag: 11.9.0-debian-10-r48 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + ## + debug: false + +## String to partially override postgresql.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override postgresql.fullname template +## +# fullnameOverride: + +## +## Init containers parameters: +## volumePermissions: Change the owner of the persist volume mountpoint to RunAsUser:fsGroup +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Init container Security Context + ## Note: the chown of the data folder is done to securityContext.runAsUser + ## and not the below volumePermissions.securityContext.runAsUser + ## When runAsUser is set to special value "auto", init container will try to chwon the + ## data folder to autodetermined user&group, using commands: `id -u`:`id -G | cut -d" " -f2` + ## "auto" is especially useful for OpenShift which has scc with dynamic userids (and 0 is not allowed). + ## You may want to use this volumePermissions.securityContext.runAsUser="auto" in combination with + ## pod securityContext.enabled=false and shmVolume.chmod.enabled=false + ## + securityContext: + runAsUser: 0 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: + +## Pod Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +securityContext: + enabled: true + fsGroup: 1001 + +## Container Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +containerSecurityContext: + enabled: true + runAsUser: 1001 + +## Pod Service Account +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + enabled: false + ## Name of an already existing service account. Setting this value disables the automatic service account creation. + # name: + +## Pod Security Policy +## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +psp: + create: false + +## Creates role for ServiceAccount +## Required for PSP +## +rbac: + create: false + +replication: + enabled: true + user: repl_user + password: repl_password + slaveReplicas: 2 + ## Set synchronous commit mode: on, off, remote_apply, remote_write and local + ## ref: https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-WAL-LEVEL + ## + synchronousCommit: "on" + ## From the number of `slaveReplicas` defined above, set the number of those that will have synchronous replication + ## NOTE: It cannot be > slaveReplicas + ## + numSynchronousReplicas: 1 + ## Replication Cluster application name. Useful for defining multiple replication policies + ## + applicationName: my_application + +## PostgreSQL admin password (used when `postgresqlUsername` is not `postgres`) +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-user-on-first-run (see note!) +# postgresqlPostgresPassword: + +## PostgreSQL user (has superuser privileges if username is `postgres`) +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run +## +postgresqlUsername: postgres + +## PostgreSQL password +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run +## +# postgresqlPassword: + +## PostgreSQL password using existing secret +## existingSecret: secret +## + +## Mount PostgreSQL secret as a file instead of passing environment variable +# usePasswordFile: false + +## Create a database +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-on-first-run +## +# postgresqlDatabase: + +## PostgreSQL data dir +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md +## +postgresqlDataDir: /bitnami/postgresql/data + +## An array to add extra environment variables +## For example: +## extraEnv: +## - name: FOO +## value: "bar" +## +# extraEnv: +extraEnv: [] + +## Name of a ConfigMap containing extra env vars +## +# extraEnvVarsCM: + +## Specify extra initdb args +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md +## +# postgresqlInitdbArgs: + +## Specify a custom location for the PostgreSQL transaction log +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md +## +# postgresqlInitdbWalDir: + +## PostgreSQL configuration +## Specify runtime configuration parameters as a dict, using camelCase, e.g. +## {"sharedBuffers": "500MB"} +## Alternatively, you can put your postgresql.conf under the files/ directory +## ref: https://www.postgresql.org/docs/current/static/runtime-config.html +## +# postgresqlConfiguration: + +## PostgreSQL extended configuration +## As above, but _appended_ to the main configuration +## Alternatively, you can put your *.conf under the files/conf.d/ directory +## https://github.com/bitnami/bitnami-docker-postgresql#allow-settings-to-be-loaded-from-files-other-than-the-default-postgresqlconf +## +# postgresqlExtendedConf: + +## Configure current cluster's master server to be the standby server in other cluster. +## This will allow cross cluster replication and provide cross cluster high availability. +## You will need to configure pgHbaConfiguration if you want to enable this feature with local cluster replication enabled. +## +masterAsStandBy: + enabled: false + # masterHost: + # masterPort: + +## PostgreSQL client authentication configuration +## Specify content for pg_hba.conf +## Default: do not create pg_hba.conf +## Alternatively, you can put your pg_hba.conf under the files/ directory +# pgHbaConfiguration: |- +# local all all trust +# host all all localhost trust +# host mydatabase mysuser 192.168.0.0/24 md5 + +## ConfigMap with PostgreSQL configuration +## NOTE: This will override postgresqlConfiguration and pgHbaConfiguration +# configurationConfigMap: + +## ConfigMap with PostgreSQL extended configuration +# extendedConfConfigMap: + +## initdb scripts +## Specify dictionary of scripts to be run at first boot +## Alternatively, you can put your scripts under the files/docker-entrypoint-initdb.d directory +## +# initdbScripts: +# my_init_script.sh: | +# #!/bin/sh +# echo "Do something." + +## Specify the PostgreSQL username and password to execute the initdb scripts +# initdbUser: +# initdbPassword: + +## ConfigMap with scripts to be run at first boot +## NOTE: This will override initdbScripts +# initdbScriptsConfigMap: + +## Secret with scripts to be run at first boot (in case it contains sensitive information) +## NOTE: This can work along initdbScripts or initdbScriptsConfigMap +# initdbScriptsSecret: + +## Optional duration in seconds the pod needs to terminate gracefully. +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods +## +# terminationGracePeriodSeconds: 30 + +## LDAP configuration +## +ldap: + enabled: false + url: "" + server: "" + port: "" + prefix: "" + suffix: "" + baseDN: "" + bindDN: "" + bind_password: + search_attr: "" + search_filter: "" + scheme: "" + tls: false + +## Audit settings +## https://github.com/bitnami/bitnami-docker-postgresql#auditing +## +audit: + ## Log client hostnames + ## + logHostname: false + ## Log connections to the server + ## + logConnections: false + ## Log disconnections + ## + logDisconnections: false + ## Operation to audit using pgAudit (default if not set) + ## + pgAuditLog: "" + ## Log catalog using pgAudit + ## + pgAuditLogCatalog: "off" + ## Log level for clients + ## + clientMinMessages: error + ## Template for log line prefix (default if not set) + ## + logLinePrefix: "" + ## Log timezone + ## + logTimezone: "" + +## Shared preload libraries +## +postgresqlSharedPreloadLibraries: "pgaudit" + +## Maximum total connections +## +postgresqlMaxConnections: + +## Maximum connections for the postgres user +## +postgresqlPostgresConnectionLimit: + +## Maximum connections for the created user +## +postgresqlDbUserConnectionLimit: + +## TCP keepalives interval +## +postgresqlTcpKeepalivesInterval: + +## TCP keepalives idle +## +postgresqlTcpKeepalivesIdle: + +## TCP keepalives count +## +postgresqlTcpKeepalivesCount: + +## Statement timeout +## +postgresqlStatementTimeout: + +## Remove pg_hba.conf lines with the following comma-separated patterns +## (cannot be used with custom pg_hba.conf) +## +postgresqlPghbaRemoveFilters: + +## PostgreSQL service configuration +## +service: + ## PosgresSQL service type + ## + type: ClusterIP + # clusterIP: None + port: 5432 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. Evaluated as a template. + ## + annotations: {} + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources. Evaluated as a template. + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + # loadBalancerSourceRanges: + # - 10.10.10.0/24 + +## Start master and slave(s) pod(s) without limitations on shm memory. +## By default docker and containerd (and possibly other container runtimes) +## limit `/dev/shm` to `64M` (see e.g. the +## [docker issue](https://github.com/docker-library/postgres/issues/416) and the +## [containerd issue](https://github.com/containerd/containerd/issues/3654), +## which could be not enough if PostgreSQL uses parallel workers heavily. +## +shmVolume: + ## Set `shmVolume.enabled` to `true` to mount a new tmpfs volume to remove + ## this limitation. + ## + enabled: true + ## Set to `true` to `chmod 777 /dev/shm` on a initContainer. + ## This option is ingored if `volumePermissions.enabled` is `false` + ## + chmod: + enabled: true + +## PostgreSQL data Persistent Volume Storage Class +## If defined, storageClassName: +## If set to "-", storageClassName: "", which disables dynamic provisioning +## If undefined (the default) or set to null, no storageClassName spec is +## set, choosing the default provisioner. (gp2 on AWS, standard on +## GKE, AWS & OpenStack) +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template, so, for example, the name can depend on .Release or .Chart + ## + # existingClaim: + + ## The path the volume will be mounted at, useful when using different + ## PostgreSQL images. + ## + mountPath: /bitnami/postgresql + + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + ## + subPath: "" + + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + +## updateStrategy for PostgreSQL StatefulSet and its slaves StatefulSets +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies +## +updateStrategy: + type: RollingUpdate + +## +## PostgreSQL Master parameters +## +master: + ## Node, affinity, tolerations, and priorityclass settings for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption + ## + nodeSelector: {} + affinity: {} + tolerations: [] + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + priorityClassName: "" + ## Additional PostgreSQL Master Volume mounts + ## + extraVolumeMounts: [] + ## Additional PostgreSQL Master Volumes + ## + extraVolumes: [] + ## Add sidecars to the pod + ## + ## For example: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + + ## Override the service configuration for master + ## + service: {} + # type: + # nodePort: + # clusterIP: + +## +## PostgreSQL Slave parameters +## +slave: + ## Node, affinity, tolerations, and priorityclass settings for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption + ## + nodeSelector: {} + affinity: {} + tolerations: [] + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + priorityClassName: "" + ## Extra init containers + ## Example + ## + ## extraInitContainers: + ## - name: do-something + ## image: busybox + ## command: ['do', 'something'] + ## + extraInitContainers: [] + ## Additional PostgreSQL Slave Volume mounts + ## + extraVolumeMounts: [] + ## Additional PostgreSQL Slave Volumes + ## + extraVolumes: [] + ## Add sidecars to the pod + ## + ## For example: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + + ## Override the service configuration for slave + ## + service: {} + # type: + # nodePort: + # clusterIP: + ## Whether to enable PostgreSQL slave replicas data Persistent + ## + persistence: + enabled: true + + # Override the resource configuration for slave + resources: {} + # requests: + # memory: 256Mi + # cpu: 250m + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 256Mi + cpu: 250m + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +networkPolicy: + ## Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port PostgreSQL is listening + ## on. When true, PostgreSQL will accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + + ## if explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the DB. + ## But sometimes, we want the DB to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## Custom Liveness probe +## +customLivenessProbe: {} + +## Custom Rediness probe +## +customReadinessProbe: {} + +## +## TLS configuration +## +tls: + # Enable TLS traffic + enabled: false + # + # Whether to use the server's TLS cipher preferences rather than the client's. + preferServerCiphers: true + # + # Name of the Secret that contains the certificates + certificatesSecret: "" + # + # Certificate filename + certFilename: "" + # + # Certificate Key filename + certKeyFilename: "" + # + # CA Certificate filename + # If provided, PostgreSQL will authenticate TLS/SSL clients by requesting them a certificate + # ref: https://www.postgresql.org/docs/9.6/auth-methods.html + certCAFilename: + # + # File containing a Certificate Revocation List + crlFilename: + +## Configure metrics exporter +## +metrics: + enabled: true + # resources: {} + service: + type: ClusterIP + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9187" + loadBalancerIP: + serviceMonitor: + enabled: false + additionalLabels: {} + # namespace: monitoring + # interval: 30s + # scrapeTimeout: 10s + ## Custom PrometheusRule to be defined + ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart + ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions + ## + prometheusRule: + enabled: false + additionalLabels: {} + namespace: "" + ## These are just examples rules, please adapt them to your needs. + ## Make sure to constraint the rules to the current postgresql service. + ## rules: + ## - alert: HugeReplicationLag + ## expr: pg_replication_lag{service="{{ template "postgresql.fullname" . }}-metrics"} / 3600 > 1 + ## for: 1m + ## labels: + ## severity: critical + ## annotations: + ## description: replication for {{ template "postgresql.fullname" . }} PostgreSQL is lagging by {{ "{{ $value }}" }} hour(s). + ## summary: PostgreSQL replication is lagging by {{ "{{ $value }}" }} hour(s). + ## + rules: [] + + image: + registry: docker.io + repository: bitnami/postgres-exporter + tag: 0.8.0-debian-10-r242 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Define additional custom metrics + ## ref: https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file + # customMetrics: + # pg_database: + # query: "SELECT d.datname AS name, CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') THEN pg_catalog.pg_database_size(d.datname) ELSE 0 END AS size FROM pg_catalog.pg_database d where datname not in ('template0', 'template1', 'postgres')" + # metrics: + # - name: + # usage: "LABEL" + # description: "Name of the database" + # - size_bytes: + # usage: "GAUGE" + # description: "Size of the database in bytes" + ## An array to add extra env vars to configure postgres-exporter + ## see: https://github.com/wrouesnel/postgres_exporter#environment-variables + ## For example: + # extraEnvVars: + # - name: PG_EXPORTER_DISABLE_DEFAULT_METRICS + # value: "true" + extraEnvVars: {} + + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + enabled: false + runAsUser: 1001 + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## Configure extra options for liveness and readiness probes + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 diff --git a/scripts/helm/helm/databases/charts/postgresql/values.schema.json b/scripts/helm/helm/databases/charts/postgresql/values.schema.json new file mode 100755 index 000000000..7b5e2efc3 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/values.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "postgresqlUsername": { + "type": "string", + "title": "Admin user", + "form": true + }, + "postgresqlPassword": { + "type": "string", + "title": "Password", + "form": true + }, + "persistence": { + "type": "object", + "properties": { + "size": { + "type": "string", + "title": "Persistent Volume Size", + "form": true, + "render": "slider", + "sliderMin": 1, + "sliderMax": 100, + "sliderUnit": "Gi" + } + } + }, + "resources": { + "type": "object", + "title": "Required Resources", + "description": "Configure resource requests", + "form": true, + "properties": { + "requests": { + "type": "object", + "properties": { + "memory": { + "type": "string", + "form": true, + "render": "slider", + "title": "Memory Request", + "sliderMin": 10, + "sliderMax": 2048, + "sliderUnit": "Mi" + }, + "cpu": { + "type": "string", + "form": true, + "render": "slider", + "title": "CPU Request", + "sliderMin": 10, + "sliderMax": 2000, + "sliderUnit": "m" + } + } + } + } + }, + "replication": { + "type": "object", + "form": true, + "title": "Replication Details", + "properties": { + "enabled": { + "type": "boolean", + "title": "Enable Replication", + "form": true + }, + "slaveReplicas": { + "type": "integer", + "title": "Slave Replicas", + "form": true, + "hidden": { + "value": false, + "path": "replication/enabled" + } + } + } + }, + "volumePermissions": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable Init Containers", + "description": "Change the owner of the persist volume mountpoint to RunAsUser:fsGroup" + } + } + }, + "metrics": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "title": "Configure metrics exporter", + "form": true + } + } + } + } +} diff --git a/scripts/helm/helm/databases/charts/postgresql/values.yaml b/scripts/helm/helm/databases/charts/postgresql/values.yaml new file mode 100755 index 000000000..c97200904 --- /dev/null +++ b/scripts/helm/helm/databases/charts/postgresql/values.yaml @@ -0,0 +1,722 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + postgresql: {} +# imageRegistry: myRegistryName +# imagePullSecrets: +# - myRegistryKeySecretName +# storageClass: myStorageClass + +## Bitnami PostgreSQL image version +## ref: https://hub.docker.com/r/bitnami/postgresql/tags/ +## +image: + registry: docker.io + repository: bitnami/postgresql + tag: 11.9.0-debian-10-r48 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Set to true if you would like to see extra information on logs + ## It turns BASH and NAMI debugging in minideb + ## ref: https://github.com/bitnami/minideb-extras/#turn-on-bash-debugging + ## + debug: false + +## String to partially override postgresql.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override postgresql.fullname template +## +# fullnameOverride: + +## +## Init containers parameters: +## volumePermissions: Change the owner of the persist volume mountpoint to RunAsUser:fsGroup +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/minideb + tag: buster + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Init container Security Context + ## Note: the chown of the data folder is done to securityContext.runAsUser + ## and not the below volumePermissions.securityContext.runAsUser + ## When runAsUser is set to special value "auto", init container will try to chwon the + ## data folder to autodetermined user&group, using commands: `id -u`:`id -G | cut -d" " -f2` + ## "auto" is especially useful for OpenShift which has scc with dynamic userids (and 0 is not allowed). + ## You may want to use this volumePermissions.securityContext.runAsUser="auto" in combination with + ## pod securityContext.enabled=false and shmVolume.chmod.enabled=false + ## + securityContext: + runAsUser: 0 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: + +## Pod Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +securityContext: + enabled: true + fsGroup: 1001 + +## Container Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +containerSecurityContext: + enabled: true + runAsUser: 1001 + +## Pod Service Account +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + enabled: false + ## Name of an already existing service account. Setting this value disables the automatic service account creation. + # name: + +## Pod Security Policy +## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +psp: + create: false + +## Creates role for ServiceAccount +## Required for PSP +## +rbac: + create: false + +replication: + enabled: false + user: repl_user + password: repl_password + slaveReplicas: 1 + ## Set synchronous commit mode: on, off, remote_apply, remote_write and local + ## ref: https://www.postgresql.org/docs/9.6/runtime-config-wal.html#GUC-WAL-LEVEL + ## + synchronousCommit: 'off' + ## From the number of `slaveReplicas` defined above, set the number of those that will have synchronous replication + ## NOTE: It cannot be > slaveReplicas + ## + numSynchronousReplicas: 0 + ## Replication Cluster application name. Useful for defining multiple replication policies + ## + applicationName: my_application + +## PostgreSQL admin password (used when `postgresqlUsername` is not `postgres`) +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-user-on-first-run (see note!) +# postgresqlPostgresPassword: + +## PostgreSQL user (has superuser privileges if username is `postgres`) +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run +## +postgresqlUsername: postgres + +## PostgreSQL password +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run +## +# postgresqlPassword: asayerPostgres + +## PostgreSQL password using existing secret +## existingSecret: secret +## + +## Mount PostgreSQL secret as a file instead of passing environment variable +# usePasswordFile: false + +## Create a database +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-on-first-run +## +# postgresqlDatabase: + +## PostgreSQL data dir +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md +## +postgresqlDataDir: /bitnami/postgresql/data + +## An array to add extra environment variables +## For example: +## extraEnv: +## - name: FOO +## value: "bar" +## +# extraEnv: +extraEnv: [] + +## Name of a ConfigMap containing extra env vars +## +# extraEnvVarsCM: + +## Specify extra initdb args +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md +## +# postgresqlInitdbArgs: + +## Specify a custom location for the PostgreSQL transaction log +## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md +## +# postgresqlInitdbWalDir: + +## PostgreSQL configuration +## Specify runtime configuration parameters as a dict, using camelCase, e.g. +## {"sharedBuffers": "500MB"} +## Alternatively, you can put your postgresql.conf under the files/ directory +## ref: https://www.postgresql.org/docs/current/static/runtime-config.html +## +# postgresqlConfiguration: + +## PostgreSQL extended configuration +## As above, but _appended_ to the main configuration +## Alternatively, you can put your *.conf under the files/conf.d/ directory +## https://github.com/bitnami/bitnami-docker-postgresql#allow-settings-to-be-loaded-from-files-other-than-the-default-postgresqlconf +## +# postgresqlExtendedConf: + +## Configure current cluster's master server to be the standby server in other cluster. +## This will allow cross cluster replication and provide cross cluster high availability. +## You will need to configure pgHbaConfiguration if you want to enable this feature with local cluster replication enabled. +## +masterAsStandBy: + enabled: false + # masterHost: + # masterPort: + +## PostgreSQL client authentication configuration +## Specify content for pg_hba.conf +## Default: do not create pg_hba.conf +## Alternatively, you can put your pg_hba.conf under the files/ directory +# pgHbaConfiguration: |- +# local all all trust +# host all all localhost trust +# host mydatabase mysuser 192.168.0.0/24 md5 + +## ConfigMap with PostgreSQL configuration +## NOTE: This will override postgresqlConfiguration and pgHbaConfiguration +# configurationConfigMap: + +## ConfigMap with PostgreSQL extended configuration +# extendedConfConfigMap: + +## initdb scripts +## Specify dictionary of scripts to be run at first boot +## Alternatively, you can put your scripts under the files/docker-entrypoint-initdb.d directory +## +# initdbScripts: +# my_init_script.sh: | +# #!/bin/sh +# echo "Do something." + +## ConfigMap with scripts to be run at first boot +## NOTE: This will override initdbScripts +# initdbScriptsConfigMap: + +## Secret with scripts to be run at first boot (in case it contains sensitive information) +## NOTE: This can work along initdbScripts or initdbScriptsConfigMap +# initdbScriptsSecret: + +## Specify the PostgreSQL username and password to execute the initdb scripts +# initdbUser: +# initdbPassword: + +## Audit settings +## https://github.com/bitnami/bitnami-docker-postgresql#auditing +## +audit: + ## Log client hostnames + ## + logHostname: false + ## Log connections to the server + ## + logConnections: false + ## Log disconnections + ## + logDisconnections: false + ## Operation to audit using pgAudit (default if not set) + ## + pgAuditLog: "" + ## Log catalog using pgAudit + ## + pgAuditLogCatalog: "off" + ## Log level for clients + ## + clientMinMessages: error + ## Template for log line prefix (default if not set) + ## + logLinePrefix: "" + ## Log timezone + ## + logTimezone: "" + +## Shared preload libraries +## +postgresqlSharedPreloadLibraries: "pgaudit" + +## Maximum total connections +## +postgresqlMaxConnections: + +## Maximum connections for the postgres user +## +postgresqlPostgresConnectionLimit: + +## Maximum connections for the created user +## +postgresqlDbUserConnectionLimit: + +## TCP keepalives interval +## +postgresqlTcpKeepalivesInterval: + +## TCP keepalives idle +## +postgresqlTcpKeepalivesIdle: + +## TCP keepalives count +## +postgresqlTcpKeepalivesCount: + +## Statement timeout +## +postgresqlStatementTimeout: + +## Remove pg_hba.conf lines with the following comma-separated patterns +## (cannot be used with custom pg_hba.conf) +## +postgresqlPghbaRemoveFilters: + +## Optional duration in seconds the pod needs to terminate gracefully. +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods +## +# terminationGracePeriodSeconds: 30 + +## LDAP configuration +## +ldap: + enabled: false + url: '' + server: '' + port: '' + prefix: '' + suffix: '' + baseDN: '' + bindDN: '' + bind_password: + search_attr: '' + search_filter: '' + scheme: '' + tls: false + +## PostgreSQL service configuration +## +service: + ## PosgresSQL service type + ## + type: ClusterIP + # clusterIP: None + port: 5432 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. Evaluated as a template. + ## + annotations: {} + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources. Evaluated as a template. + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + # loadBalancerSourceRanges: + # - 10.10.10.0/24 + +## Start master and slave(s) pod(s) without limitations on shm memory. +## By default docker and containerd (and possibly other container runtimes) +## limit `/dev/shm` to `64M` (see e.g. the +## [docker issue](https://github.com/docker-library/postgres/issues/416) and the +## [containerd issue](https://github.com/containerd/containerd/issues/3654), +## which could be not enough if PostgreSQL uses parallel workers heavily. +## +shmVolume: + ## Set `shmVolume.enabled` to `true` to mount a new tmpfs volume to remove + ## this limitation. + ## + enabled: true + ## Set to `true` to `chmod 777 /dev/shm` on a initContainer. + ## This option is ingored if `volumePermissions.enabled` is `false` + ## + chmod: + enabled: true + +## PostgreSQL data Persistent Volume Storage Class +## If defined, storageClassName: +## If set to "-", storageClassName: "", which disables dynamic provisioning +## If undefined (the default) or set to null, no storageClassName spec is +## set, choosing the default provisioner. (gp2 on AWS, standard on +## GKE, AWS & OpenStack) +## +persistence: + enabled: true + ## A manually managed Persistent Volume and Claim + ## If defined, PVC must be created manually before volume will be bound + ## The value is evaluated as a template, so, for example, the name can depend on .Release or .Chart + ## + # existingClaim: + + ## The path the volume will be mounted at, useful when using different + ## PostgreSQL images. + ## + mountPath: /bitnami/postgresql + + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + ## + subPath: '' + + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + +## updateStrategy for PostgreSQL StatefulSet and its slaves StatefulSets +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies +## +updateStrategy: + type: RollingUpdate + +## +## PostgreSQL Master parameters +## +master: + ## Node, affinity, tolerations, and priorityclass settings for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption + ## + nodeSelector: {} + affinity: {} + tolerations: [] + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + priorityClassName: '' + ## Extra init containers + ## Example + ## + ## extraInitContainers: + ## - name: do-something + ## image: busybox + ## command: ['do', 'something'] + ## + extraInitContainers: [] + + ## Additional PostgreSQL Master Volume mounts + ## + extraVolumeMounts: [] + ## Additional PostgreSQL Master Volumes + ## + extraVolumes: [] + ## Add sidecars to the pod + ## + ## For example: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + + ## Override the service configuration for master + ## + service: {} + # type: + # nodePort: + # clusterIP: + +## +## PostgreSQL Slave parameters +## +slave: + ## Node, affinity, tolerations, and priorityclass settings for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption + ## + nodeSelector: {} + affinity: {} + tolerations: [] + labels: {} + annotations: {} + podLabels: {} + podAnnotations: {} + priorityClassName: '' + extraInitContainers: | + # - name: do-something + # image: busybox + # command: ['do', 'something'] + ## Additional PostgreSQL Slave Volume mounts + ## + extraVolumeMounts: [] + ## Additional PostgreSQL Slave Volumes + ## + extraVolumes: [] + ## Add sidecars to the pod + ## + ## For example: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] + + ## Override the service configuration for slave + ## + service: {} + # type: + # nodePort: + # clusterIP: + + ## Whether to enable PostgreSQL slave replicas data Persistent + ## + persistence: + enabled: true + + # Override the resource configuration for slave + resources: {} + # requests: + # memory: 256Mi + # cpu: 250m + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 256Mi + cpu: 250m + +## Add annotations to all the deployed resources +## +commonAnnotations: {} + +networkPolicy: + ## Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port PostgreSQL is listening + ## on. When true, PostgreSQL will accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + + ## if explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the DB. + ## But sometimes, we want the DB to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} + +## Configure extra options for liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## Custom Liveness probe +## +customLivenessProbe: {} + +## Custom Rediness probe +## +customReadinessProbe: {} + +## +## TLS configuration +## +tls: + # Enable TLS traffic + enabled: false + # + # Whether to use the server's TLS cipher preferences rather than the client's. + preferServerCiphers: true + # + # Name of the Secret that contains the certificates + certificatesSecret: '' + # + # Certificate filename + certFilename: '' + # + # Certificate Key filename + certKeyFilename: '' + # + # CA Certificate filename + # If provided, PostgreSQL will authenticate TLS/SSL clients by requesting them a certificate + # ref: https://www.postgresql.org/docs/9.6/auth-methods.html + certCAFilename: + # + # File containing a Certificate Revocation List + crlFilename: + +## Configure metrics exporter +## +metrics: + enabled: false + # resources: {} + service: + type: ClusterIP + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9187' + loadBalancerIP: + serviceMonitor: + enabled: false + additionalLabels: {} + # namespace: monitoring + # interval: 30s + # scrapeTimeout: 10s + ## Custom PrometheusRule to be defined + ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart + ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions + ## + prometheusRule: + enabled: false + additionalLabels: {} + namespace: '' + ## These are just examples rules, please adapt them to your needs. + ## Make sure to constraint the rules to the current postgresql service. + ## rules: + ## - alert: HugeReplicationLag + ## expr: pg_replication_lag{service="{{ template "postgresql.fullname" . }}-metrics"} / 3600 > 1 + ## for: 1m + ## labels: + ## severity: critical + ## annotations: + ## description: replication for {{ template "postgresql.fullname" . }} PostgreSQL is lagging by {{ "{{ $value }}" }} hour(s). + ## summary: PostgreSQL replication is lagging by {{ "{{ $value }}" }} hour(s). + ## + rules: [] + + image: + registry: docker.io + repository: bitnami/postgres-exporter + tag: 0.8.0-debian-10-r242 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + ## Define additional custom metrics + ## ref: https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file + # customMetrics: + # pg_database: + # query: "SELECT d.datname AS name, CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') THEN pg_catalog.pg_database_size(d.datname) ELSE 0 END AS size_bytes FROM pg_catalog.pg_database d where datname not in ('template0', 'template1', 'postgres')" + # metrics: + # - name: + # usage: "LABEL" + # description: "Name of the database" + # - size_bytes: + # usage: "GAUGE" + # description: "Size of the database in bytes" + # + ## An array to add extra env vars to configure postgres-exporter + ## see: https://github.com/wrouesnel/postgres_exporter#environment-variables + ## For example: + # extraEnvVars: + # - name: PG_EXPORTER_DISABLE_DEFAULT_METRICS + # value: "true" + extraEnvVars: {} + + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + enabled: false + runAsUser: 1001 + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## Configure extra options for liveness and readiness probes + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + successThreshold: 1 + +## Array with extra yaml to deploy with the chart. Evaluated as a template +## +extraDeploy: [] diff --git a/scripts/helm/helm/databases/charts/redis/.helmignore b/scripts/helm/helm/databases/charts/redis/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/scripts/helm/helm/databases/charts/redis/Chart.lock b/scripts/helm/helm/databases/charts/redis/Chart.lock new file mode 100644 index 000000000..69cc5b92f --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.4.2 +digest: sha256:4e3ec38e0e27e9fc1defb2a13f67a0aa12374bf0b15f06a6c13b1b46df6bffeb +generated: "2021-04-05T11:40:59.141264592Z" diff --git a/scripts/helm/helm/databases/charts/redis/Chart.yaml b/scripts/helm/helm/databases/charts/redis/Chart.yaml new file mode 100644 index 000000000..dd36c43e0 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/Chart.yaml @@ -0,0 +1,29 @@ +annotations: + category: Database +apiVersion: v2 +appVersion: 6.0.12 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Open source, advanced key-value store. It is often referred to as a data + structure server since keys can contain strings, hashes, lists, sets and sorted + sets. +home: https://github.com/bitnami/charts/tree/master/bitnami/redis +icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png +keywords: +- redis +- keyvalue +- database +maintainers: +- email: containers@bitnami.com + name: Bitnami +- email: cedric@desaintmartin.fr + name: desaintmartin +name: redis +sources: +- https://github.com/bitnami/bitnami-docker-redis +- http://redis.io/ +version: 12.10.1 diff --git a/scripts/helm/helm/databases/charts/redis/README.md b/scripts/helm/helm/databases/charts/redis/README.md new file mode 100644 index 000000000..de0a04e18 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/README.md @@ -0,0 +1,738 @@ +# RedisTM Chart packaged by Bitnami + +[RedisTM](http://redis.io/) is an advanced key-value cache and store. It is often referred to as a data structure server since keys can contain strings, hashes, lists, sets, sorted sets, bitmaps and hyperloglogs. + +Disclaimer: REDIS® is a registered trademark of Redis Labs Ltd.Any rights therein are reserved to Redis Labs Ltd. Any use by Bitnami is for referential purposes only and does not indicate any sponsorship, endorsement, or affiliation between Redis Labs Ltd. + +## TL;DR + +```bash +$ helm repo add bitnami https://charts.bitnami.com/bitnami +$ helm install my-release bitnami/redis +``` + +## Introduction + +This chart bootstraps a [RedisTM](https://github.com/bitnami/bitnami-docker-redis) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This chart has been tested to work with NGINX Ingress, cert-manager, fluentd and Prometheus on top of the [BKPR](https://kubeprod.io/). + +### Choose between RedisTM Helm Chart and RedisTM Cluster Helm Chart + +You can choose any of the two RedisTM Helm charts for deploying a RedisTM cluster. +While [RedisTM Helm Chart](https://github.com/bitnami/charts/tree/master/bitnami/redis) will deploy a master-slave cluster using RedisTM Sentinel, the [RedisTM Cluster Helm Chart](https://github.com/bitnami/charts/tree/master/bitnami/redis-cluster) will deploy a RedisTM Cluster topology with sharding. +The main features of each chart are the following: + +| RedisTM | RedisTM Cluster | +|--------------------------------------------------------|------------------------------------------------------------------------| +| Supports multiple databases | Supports only one database. Better if you have a big dataset | +| Single write point (single master) | Multiple write points (multiple masters) | +| ![RedisTM Topology](img/redis-topology.png) | ![RedisTM Cluster Topology](img/redis-cluster-topology.png) | + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 3.1.0 +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ helm install my-release bitnami/redis +``` + +The command deploys RedisTM on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```bash +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Parameters + +The following table lists the configurable parameters of the RedisTM chart and their default values. + +| Parameter | Description | Default | +|:------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------| +| `global.imageRegistry` | Global Docker image registry | `nil` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` (does not add image pull secrets to deployed pods) | +| `global.storageClass` | Global storage class for dynamic provisioning | `nil` | +| `global.redis.password` | RedisTM password (overrides `password`) | `nil` | +| `image.registry` | RedisTM Image registry | `docker.io` | +| `image.repository` | RedisTM Image name | `bitnami/redis` | +| `image.tag` | RedisTM Image tag | `{TAG_NAME}` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Specify docker-registry secret names as an array | `nil` | +| `nameOverride` | String to partially override redis.fullname template with a string (will prepend the release name) | `nil` | +| `fullnameOverride` | String to fully override redis.fullname template with a string | `nil` | +| `cluster.enabled` | Use master-slave topology | `true` | +| `cluster.slaveCount` | Number of slaves | `2` | +| `existingSecret` | Name of existing secret object (for password authentication) | `nil` | +| `existingSecretPasswordKey` | Name of key containing password to be retrieved from the existing secret | `nil` | +| `usePassword` | Use password | `true` | +| `usePasswordFile` | Mount passwords as files instead of environment variables | `false` | +| `password` | RedisTM password (ignored if existingSecret set) | Randomly generated | +| `configmap` | Additional common RedisTM node configuration (this value is evaluated as a template) | See values.yaml | +| `clusterDomain` | Kubernetes DNS Domain name to use | `cluster.local` | +| `extraVolumes` | Array of extra volumes to be added to master & slave nodes (evaluated as a template) | `[]` | +| `extraVolumeMounts` | Array of extra volume mounts to be added to master & slave nodes (evaluated as a template) | `[]` | +| `networkPolicy.enabled` | Enable NetworkPolicy | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.ingressNSMatchLabels` | Allow connections from other namespaces | `{}` | +| `networkPolicy.ingressNSPodMatchLabels` | For other namespaces match by pod labels and namespace labels | `{}` | +| `securityContext.*` | Other pod security context to be included as-is in the pod spec | `{}` | +| `securityContext.enabled` | Enable security context (both redis master and slave pods) | `true` | +| `securityContext.fsGroup` | Group ID for the container (both redis master and slave pods) | `1001` | +| `containerSecurityContext.*` | Other container security context to be included as-is in the container spec | `{}` | +| `containerSecurityContext.enabled` | Enable security context (both redis master and slave containers) | `true` | +| `containerSecurityContext.runAsUser` | User ID for the container (both redis master and slave containers) | `1001` | +| `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `false` | +| `serviceAccount.name` | The name of the ServiceAccount to create | Generated using the fullname template | +| `serviceAccount.annotations` | Specifies annotations to add to ServiceAccount. | `nil` | +| `rbac.create` | Specifies whether RBAC resources should be created | `false` | +| `rbac.role.rules` | Rules to create | `[]` | +| `metrics.enabled` | Start a side-car prometheus exporter | `false` | +| `metrics.image.registry` | RedisTM exporter image registry | `docker.io` | +| `metrics.image.repository` | RedisTM exporter image name | `bitnami/redis-exporter` | +| `metrics.image.tag` | RedisTM exporter image tag | `{TAG_NAME}` | +| `metrics.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `metrics.image.pullSecrets` | Specify docker-registry secret names as an array | `nil` | +| `metrics.extraArgs` | Extra arguments for the binary; possible values [here](https://github.com/oliver006/redis_exporter#flags) | {} | +| `metrics.podLabels` | Additional labels for Metrics exporter pod | {} | +| `metrics.podAnnotations` | Additional annotations for Metrics exporter pod | {} | +| `metrics.resources` | Exporter resource requests/limit | Memory: `256Mi`, CPU: `100m` | +| `metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (also requires `metrics.enabled` to be `true`) | `false` | +| `metrics.serviceMonitor.namespace` | Optional namespace which Prometheus is running in | `nil` | +| `metrics.serviceMonitor.interval` | How frequently to scrape metrics (use by default, falling back to Prometheus' default) | `nil` | +| `metrics.serviceMonitor.selector` | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install | `{ prometheus: kube-prometheus }` | +| `metrics.serviceMonitor.relabelings` | ServiceMonitor relabelings. Value is evaluated as a template | `[]` | +| `metrics.serviceMonitor.metricRelabelings` | ServiceMonitor metricRelabelings. Value is evaluated as a template | `[]` | +| `metrics.service.type` | Kubernetes Service type (redis metrics) | `ClusterIP` | +| `metrics.service.externalTrafficPolicy` | External traffic policy (when service type is LoadBalancer) | `Cluster` | +| `metrics.service.annotations` | Annotations for the services to monitor (redis master and redis slave service) | {} | +| `metrics.service.labels` | Additional labels for the metrics service | {} | +| `metrics.service.loadBalancerIP` | loadBalancerIP if redis metrics service type is `LoadBalancer` | `nil` | +| `metrics.priorityClassName` | Metrics exporter pod priorityClassName | `nil` | +| `metrics.prometheusRule.enabled` | Set this to true to create prometheusRules for Prometheus operator | `false` | +| `metrics.prometheusRule.additionalLabels` | Additional labels that can be used so prometheusRules will be discovered by Prometheus | `{}` | +| `metrics.prometheusRule.namespace` | namespace where prometheusRules resource should be created | Same namespace as redis | +| `metrics.prometheusRule.rules` | [rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) to be created, check values for an example. | `[]` | +| `metrics.redisTargetHost` | way to specify an alternative redis hostname, if you set a local endpoint in hostAliases to match specific redis server certificate CN/SAN for example. + | `localhost` | +| `persistence.existingClaim` | Provide an existing PersistentVolumeClaim | `nil` | +| `master.persistence.enabled` | Use a PVC to persist data (master node) | `true` | +| `master.hostAliases` | Add deployment host aliases | `[]` | +| `master.persistence.path` | Path to mount the volume at, to use other images | `/data` | +| `master.persistence.subPath` | Subdirectory of the volume to mount at | `""` | +| `master.persistence.storageClass` | Storage class of backing PVC | `generic` | +| `master.persistence.accessModes` | Persistent Volume Access Modes | `[ReadWriteOnce]` | +| `master.persistence.size` | Size of data volume | `8Gi` | +| `master.persistence.matchLabels` | matchLabels persistent volume selector | `{}` | +| `master.persistence.matchExpressions` | matchExpressions persistent volume selector | `{}` | +| `master.persistence.volumes` | Additional volumes without creating PVC | `{}` | +| `master.statefulset.labels` | Additional labels for redis master StatefulSet | `{}` | +| `master.statefulset.annotations` | Additional annotations for redis master StatefulSet | `{}` | +| `master.statefulset.updateStrategy` | Update strategy for StatefulSet | onDelete | +| `master.statefulset.rollingUpdatePartition` | Partition update strategy | `nil` | +| `master.statefulset.volumeClaimTemplates.labels` | Additional labels for redis master StatefulSet volumeClaimTemplates | `{}` | +| `master.statefulset.volumeClaimTemplates.annotations` | Additional annotations for redis master StatefulSet volumeClaimTemplates | `{}` | +| `master.podLabels` | Additional labels for RedisTM master pod | {} | +| `master.podAnnotations` | Additional annotations for RedisTM master pod | {} | +| `master.extraEnvVars` | Additional Environment Variables passed to the pod of the master's stateful set set | `[]` | +| `master.extraEnvVarCMs` | Additional Environment Variables ConfigMappassed to the pod of the master's stateful set set | `[]` | +| `master.extraEnvVarsSecret` | Additional Environment Variables Secret passed to the master's stateful set | `[]` | +| `master.extraVolumes` | Array of extra volumes to be added to master pod (evaluated as a template) | `[]` | +| `master.extraVolumeMounts` | Array of extra volume mounts to be added to master pod (evaluated as a template) | `[]` | +| `podDisruptionBudget.enabled` | Pod Disruption Budget toggle | `false` | +| `podDisruptionBudget.minAvailable` | Minimum available pods | `1` | +| `podDisruptionBudget.maxUnavailable` | Maximum unavailable | `nil` | +| `redisPort` | RedisTM port (in both master and slaves) | `6379` | +| `tls.enabled` | Enable TLS support for replication traffic | `false` | +| `tls.authClients` | Require clients to authenticate or not | `true` | +| `tls.certificatesSecret` | Name of the secret that contains the certificates | `nil` | +| `tls.certFilename` | Certificate filename | `nil` | +| `tls.certKeyFilename` | Certificate key filename | `nil` | +| `tls.certCAFilename` | CA Certificate filename | `nil` | +| `tls.dhParamsFilename` | DH params (in order to support DH based ciphers) | `nil` | +| `master.command` | RedisTM master entrypoint string. The command `redis-server` is executed if this is not provided. Note this is prepended with `exec` | `/run.sh` | +| `master.preExecCmds` | Text to inset into the startup script immediately prior to `master.command`. Use this if you need to run other ad-hoc commands as part of startup | `nil` | +| `master.configmap` | Additional RedisTM configuration for the master nodes (this value is evaluated as a template) | `nil` | +| `master.disableCommands` | Array of RedisTM commands to disable (master) | `["FLUSHDB", "FLUSHALL"]` | +| `master.extraFlags` | RedisTM master additional command line flags | [] | +| `master.nodeSelector` | RedisTM master Node labels for pod assignment | {"beta.kubernetes.io/arch": "amd64"} | +| `master.tolerations` | Toleration labels for RedisTM master pod assignment | [] | +| `master.affinity` | Affinity settings for RedisTM master pod assignment | {} | +| `master.schedulerName` | Name of an alternate scheduler | `nil` | +| `master.service.type` | Kubernetes Service type (redis master) | `ClusterIP` | +| `master.service.externalTrafficPolicy` | External traffic policy (when service type is LoadBalancer) | `Cluster` | +| `master.service.port` | Kubernetes Service port (redis master) | `6379` | +| `master.service.nodePort` | Kubernetes Service nodePort (redis master) | `nil` | +| `master.service.annotations` | annotations for redis master service | {} | +| `master.service.labels` | Additional labels for redis master service | {} | +| `master.service.loadBalancerIP` | loadBalancerIP if redis master service type is `LoadBalancer` | `nil` | +| `master.service.loadBalancerSourceRanges` | loadBalancerSourceRanges if redis master service type is `LoadBalancer` | `nil` | +| `master.resources` | RedisTM master CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `100m` | +| `master.livenessProbe.enabled` | Turn on and off liveness probe (redis master pod) | `true` | +| `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated (redis master pod) | `5` | +| `master.livenessProbe.periodSeconds` | How often to perform the probe (redis master pod) | `5` | +| `master.livenessProbe.timeoutSeconds` | When the probe times out (redis master pod) | `5` | +| `master.livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed (redis master pod) | `1` | +| `master.livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | `5` | +| `master.readinessProbe.enabled` | Turn on and off readiness probe (redis master pod) | `true` | +| `master.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated (redis master pod) | `5` | +| `master.readinessProbe.periodSeconds` | How often to perform the probe (redis master pod) | `5` | +| `master.readinessProbe.timeoutSeconds` | When the probe times out (redis master pod) | `1` | +| `master.readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed (redis master pod) | `1` | +| `master.readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | `5` | +| `master.shareProcessNamespace` | RedisTM Master pod `shareProcessNamespace` option. Enables /pause reap zombie PIDs. | `false` | +| `master.priorityClassName` | RedisTM Master pod priorityClassName | `nil` | +| `volumePermissions.enabled` | Enable init container that changes volume permissions in the registry (for cases where the default k8s `runAsUser` and `fsUser` values do not work) | `false` | +| `volumePermissions.image.registry` | Init container volume-permissions image registry | `docker.io` | +| `volumePermissions.image.repository` | Init container volume-permissions image name | `bitnami/bitnami-shell` | +| `volumePermissions.image.tag` | Init container volume-permissions image tag | `"10"` | +| `volumePermissions.image.pullPolicy` | Init container volume-permissions image pull policy | `Always` | +| `volumePermissions.resources ` | Init container volume-permissions CPU/Memory resource requests/limits | {} | +| `volumePermissions.securityContext.*` | Security context of the init container | `{}` | +| `volumePermissions.securityContext.runAsUser` | UserID for the init container (when facing issues in OpenShift or uid unknown, try value "auto") | 0 | +| `slave.hostAliases` | Add deployment host aliases | `[]` | +| `slave.service.type` | Kubernetes Service type (redis slave) | `ClusterIP` | +| `slave.service.externalTrafficPolicy` | External traffic policy (when service type is LoadBalancer) | `Cluster` | +| `slave.service.nodePort` | Kubernetes Service nodePort (redis slave) | `nil` | +| `slave.service.annotations` | annotations for redis slave service | {} | +| `slave.service.labels` | Additional labels for redis slave service | {} | +| `slave.service.port` | Kubernetes Service port (redis slave) | `6379` | +| `slave.service.loadBalancerIP` | LoadBalancerIP if RedisTM slave service type is `LoadBalancer` | `nil` | +| `slave.service.loadBalancerSourceRanges` | loadBalancerSourceRanges if RedisTM slave service type is `LoadBalancer` | `nil` | +| `slave.command` | RedisTM slave entrypoint string. The command `redis-server` is executed if this is not provided. Note this is prepended with `exec` | `/run.sh` | +| `slave.preExecCmds` | Text to inset into the startup script immediately prior to `slave.command`. Use this if you need to run other ad-hoc commands as part of startup | `nil` | +| `slave.configmap` | Additional RedisTM configuration for the slave nodes (this value is evaluated as a template) | `nil` | +| `slave.disableCommands` | Array of RedisTM commands to disable (slave) | `[FLUSHDB, FLUSHALL]` | +| `slave.extraFlags` | RedisTM slave additional command line flags | `[]` | +| `slave.livenessProbe.enabled` | Turn on and off liveness probe (redis slave pod) | `true` | +| `slave.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated (redis slave pod) | `5` | +| `slave.livenessProbe.periodSeconds` | How often to perform the probe (redis slave pod) | `5` | +| `slave.livenessProbe.timeoutSeconds` | When the probe times out (redis slave pod) | `5` | +| `slave.livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed (redis slave pod) | `1` | +| `slave.livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | `5` | +| `slave.readinessProbe.enabled` | Turn on and off slave.readiness probe (redis slave pod) | `true` | +| `slave.readinessProbe.initialDelaySeconds` | Delay before slave.readiness probe is initiated (redis slave pod) | `5` | +| `slave.readinessProbe.periodSeconds` | How often to perform the probe (redis slave pod) | `5` | +| `slave.readinessProbe.timeoutSeconds` | When the probe times out (redis slave pod) | `1` | +| `slave.readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed (redis slave pod) | `1` | +| `slave.readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. (redis slave pod) | `5` | +| `slave.shareProcessNamespace` | RedisTM slave pod `shareProcessNamespace` option. Enables /pause reap zombie PIDs. | `false` | +| `slave.persistence.enabled` | Use a PVC to persist data (slave node) | `true` | +| `slave.persistence.path` | Path to mount the volume at, to use other images | `/data` | +| `slave.persistence.subPath` | Subdirectory of the volume to mount at | `""` | +| `slave.persistence.storageClass` | Storage class of backing PVC | `generic` | +| `slave.persistence.accessModes` | Persistent Volume Access Modes | `[ReadWriteOnce]` | +| `slave.persistence.size` | Size of data volume | `8Gi` | +| `slave.persistence.matchLabels` | matchLabels persistent volume selector | `{}` | +| `slave.persistence.matchExpressions` | matchExpressions persistent volume selector | `{}` | +| `slave.statefulset.labels` | Additional labels for redis slave StatefulSet | `{}` | +| `slave.statefulset.annotations` | Additional annotations for redis slave StatefulSet | `{}` | +| `slave.statefulset.updateStrategy` | Update strategy for StatefulSet | onDelete | +| `slave.statefulset.rollingUpdatePartition` | Partition update strategy | `nil` | +| `slave.statefulset.volumeClaimTemplates.labels` | Additional labels for redis slave StatefulSet volumeClaimTemplates | `{}` | +| `slave.statefulset.volumeClaimTemplates.annotations` | Additional annotations for redis slave StatefulSet volumeClaimTemplates | `{}` | +| `slave.extraEnvVars` | Additional Environment Variables passed to the pod of the slave's stateful set set | `[]` | +| `slave.extraEnvVarCMs` | Additional Environment Variables ConfigMappassed to the pod of the slave's stateful set set | `[]` | +| `masslaveter.extraEnvVarsSecret` | Additional Environment Variables Secret passed to the slave's stateful set | `[]` | +| `slave.extraVolumes` | Array of extra volumes to be added to slave pod (evaluated as a template) | `[]` | +| `slave.extraVolumeMounts` | Array of extra volume mounts to be added to slave pod (evaluated as a template) | `[]` | +| `slave.podLabels` | Additional labels for RedisTM slave pod | `master.podLabels` | +| `slave.podAnnotations` | Additional annotations for RedisTM slave pod | `master.podAnnotations` | +| `slave.schedulerName` | Name of an alternate scheduler | `nil` | +| `slave.resources` | RedisTM slave CPU/Memory resource requests/limits | `{}` | +| `slave.affinity` | Enable node/pod affinity for slaves | {} | +| `slave.tolerations` | Toleration labels for RedisTM slave pod assignment | [] | +| `slave.spreadConstraints` | [Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for RedisTM slave pod | {} | +| `slave.priorityClassName` | RedisTM Slave pod priorityClassName | `nil` | +| `sentinel.enabled` | Enable sentinel containers | `false` | +| `sentinel.usePassword` | Use password for sentinel containers | `true` | +| `sentinel.masterSet` | Name of the sentinel master set | `mymaster` | +| `sentinel.initialCheckTimeout` | Timeout for querying the redis sentinel service for the active sentinel list | `5` | +| `sentinel.quorum` | Quorum for electing a new master | `2` | +| `sentinel.downAfterMilliseconds` | Timeout for detecting a RedisTM node is down | `60000` | +| `sentinel.failoverTimeout` | Timeout for performing a election failover | `18000` | +| `sentinel.parallelSyncs` | Number of parallel syncs in the cluster | `1` | +| `sentinel.port` | RedisTM Sentinel port | `26379` | +| `sentinel.cleanDelaySeconds` | Delay seconds before issuing the the cleaning in the next node | `5` | +| `sentinel.configmap` | Additional RedisTM configuration for the sentinel nodes (this value is evaluated as a template) | `nil` | +| `sentinel.staticID` | Enable static IDs for sentinel replicas (If disabled IDs will be randomly generated on startup) | `false` | +| `sentinel.service.type` | Kubernetes Service type (redis sentinel) | `ClusterIP` | +| `sentinel.service.externalTrafficPolicy` | External traffic policy (when service type is LoadBalancer) | `Cluster` | +| `sentinel.service.nodePort` | Kubernetes Service nodePort (redis sentinel) | `nil` | +| `sentinel.service.annotations` | annotations for redis sentinel service | {} | +| `sentinel.service.labels` | Additional labels for redis sentinel service | {} | +| `sentinel.service.redisPort` | Kubernetes Service port for RedisTM read only operations | `6379` | +| `sentinel.service.sentinelPort` | Kubernetes Service port for RedisTM sentinel | `26379` | +| `sentinel.service.redisNodePort` | Kubernetes Service node port for RedisTM read only operations | `` | +| `sentinel.service.sentinelNodePort` | Kubernetes Service node port for RedisTM sentinel | `` | +| `sentinel.service.loadBalancerIP` | LoadBalancerIP if RedisTM sentinel service type is `LoadBalancer` | `nil` | +| `sentinel.livenessProbe.enabled` | Turn on and off liveness probe (redis sentinel pod) | `true` | +| `sentinel.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated (redis sentinel pod) | `5` | +| `sentinel.livenessProbe.periodSeconds` | How often to perform the probe (redis sentinel container) | `5` | +| `sentinel.livenessProbe.timeoutSeconds` | When the probe times out (redis sentinel container) | `5` | +| `sentinel.livenessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed (redis sentinel container) | `1` | +| `sentinel.livenessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. | `5` | +| `sentinel.readinessProbe.enabled` | Turn on and off sentinel.readiness probe (redis sentinel pod) | `true` | +| `sentinel.readinessProbe.initialDelaySeconds` | Delay before sentinel.readiness probe is initiated (redis sentinel pod) | `5` | +| `sentinel.readinessProbe.periodSeconds` | How often to perform the probe (redis sentinel pod) | `5` | +| `sentinel.readinessProbe.timeoutSeconds` | When the probe times out (redis sentinel container) | `1` | +| `sentinel.readinessProbe.successThreshold` | Minimum consecutive successes for the probe to be considered successful after having failed (redis sentinel container) | `1` | +| `sentinel.readinessProbe.failureThreshold` | Minimum consecutive failures for the probe to be considered failed after having succeeded. (redis sentinel container) | `5` | +| `sentinel.resources` | RedisTM sentinel CPU/Memory resource requests/limits | `{}` | +| `sentinel.image.registry` | RedisTM Sentinel Image registry | `docker.io` | +| `sentinel.image.repository` | RedisTM Sentinel Image name | `bitnami/redis-sentinel` | +| `sentinel.image.tag` | RedisTM Sentinel Image tag | `{TAG_NAME}` | +| `sentinel.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `sentinel.image.pullSecrets` | Specify docker-registry secret names as an array | `nil` | +| `sentinel.extraEnvVars` | Additional Environment Variables passed to the pod of the sentinel node stateful set set | `[]` | +| `sentinel.extraEnvVarCMs` | Additional Environment Variables ConfigMappassed to the pod of the sentinel node stateful set set | `[]` | +| `sentinel.extraEnvVarsSecret` | Additional Environment Variables Secret passed to the sentinel node statefulset | `[]` | +| `sentinel.extraVolumes` | Array of extra volumes to be added to sentinel node (evaluated as a template) | `[]` | +| `sentinel.extraVolumeMounts` | Array of extra volume mounts to be added to sentinel node (evaluated as a template) | `[]` | +| `sentinel.preExecCmds` | Text to inset into the startup script immediately prior to `sentinel.command`. Use this if you need to run other ad-hoc commands as part of startup | `nil` | +| `sentinel.metrics.enabled` | Start a side-car prometheus sentinel exporter | `false` | +| `sentinel.metrics.image.registry` | Redis Sentinel exporter image registry | `docker.io` | +| `sentinel.metrics.image.repository` | Redis Sentinel exporter image name | `bitnami/redis-sentinel-exporter` | +| `sentinel.metrics.image.tag` | Redis Sentinel exporter image tag | `{TAG_NAME}` | +| `sentinel.metrics.image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `sentinel.metrics.image.pullSecrets` | Specify docker-registry secret names as an array | `[]` | +| `sentinel.metrics.extraArgs` | Extra arguments for the binary; possible values [here](https://github.com/leominov/redis_sentinel_exporter#configuration) | `{}` | +| `sentinel.metrics.resources` | Exporter resource requests/limit | `{}` | +| `sentinel.metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (also requires `sentinel.metrics.enabled` to be `true`) | `false` | +| `sentinel.metrics.serviceMonitor.namespace` | Optional namespace which Prometheus is running in | `nil` | +| `sentinel.metrics.serviceMonitor.interval` | How frequently to scrape metrics (use by default, falling back to Prometheus' default) | `nil` | +| `sentinel.metrics.serviceMonitor.selector` | Default to kube-prometheus install (CoreOS recommended), but should be set according to Prometheus install | `{ prometheus: kube-prometheus }` | +| `sentinel.metrics.service.type` | Kubernetes Service type (redis sentinel metrics) | `ClusterIP` | +| `sentinel.metrics.service.port` | Kubernetes service port (redis sentinel metrics) | `9355` | +| `sentinel.metrics.service.externalTrafficPolicy` | External traffic policy (when service type is LoadBalancer) | `Cluster` | +| `sentinel.metrics.service.annotations` | Annotations for the services to monitor (redis sentinel metrics service) | {} | +| `sentinel.metrics.service.labels` | Additional labels for the Sentinel metrics service | {} | +| `sentinel.metrics.service.loadBalancerIP` | loadBalancerIP if redis metrics service type is `LoadBalancer` | `nil` | +| `sentinel.metrics.priorityClassName` | Sentinel metrics exporter pod priorityClassName | `nil` | +| `sysctlImage.enabled` | Enable an init container to modify Kernel settings | `false` | +| `sysctlImage.command` | sysctlImage command to execute | [] | +| `sysctlImage.registry` | sysctlImage Init container registry | `docker.io` | +| `sysctlImage.repository` | sysctlImage Init container name | `bitnami/bitnami-shell` | +| `sysctlImage.tag` | sysctlImage Init container tag | `"10"` | +| `sysctlImage.pullPolicy` | sysctlImage Init container pull policy | `Always` | +| `sysctlImage.mountHostSys` | Mount the host `/sys` folder to `/host-sys` | `false` | +| `sysctlImage.resources` | sysctlImage Init container CPU/Memory resource requests/limits | {} | +| `podSecurityPolicy.create` | Specifies whether a PodSecurityPolicy should be created | `false` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```bash +$ helm install my-release \ + --set password=secretpassword \ + bitnami/redis +``` + +The above command sets the RedisTM server password to `secretpassword`. + +> NOTE: Once this chart is deployed, it is not possible to change the application's access credentials, such as usernames or passwords, using Helm. To change these application credentials after deployment, delete any persistent volumes (PVs) used by the chart and re-deploy it, or use the application's built-in administrative tools if available. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```bash +$ helm install my-release -f values.yaml bitnami/redis +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +> **Note for minikube users**: Current versions of minikube (v0.24.1 at the time of writing) provision `hostPath` persistent volumes that are only writable by root. Using chart defaults cause pod failure for the RedisTM pod as it attempts to write to the `/bitnami` directory. Consider installing RedisTM with `--set persistence.enabled=false`. See minikube issue [1990](https://github.com/kubernetes/minikube/issues/1990) for more information. + +## Configuration and installation details + +### [Rolling VS Immutable tags](https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### Change RedisTM version + +To modify the RedisTM version used in this chart you can specify a [valid image tag](https://hub.docker.com/r/bitnami/redis/tags/) using the `image.tag` parameter. For example, `image.tag=X.Y.Z`. This approach is also applicable to other images like exporters. + +### Cluster topologies + +#### Default: Master-Slave + +When installing the chart with `cluster.enabled=true`, it will deploy a RedisTM master StatefulSet (only one master node allowed) and a RedisTM slave StatefulSet. The slaves will be read-replicas of the master. Two services will be exposed: + + - RedisTM Master service: Points to the master, where read-write operations can be performed + - RedisTM Slave service: Points to the slaves, where only read operations are allowed. + +In case the master crashes, the slaves will wait until the master node is respawned again by the Kubernetes Controller Manager. + +#### Master-Slave with Sentinel + +When installing the chart with `cluster.enabled=true` and `sentinel.enabled=true`, it will deploy a RedisTM master StatefulSet (only one master allowed) and a RedisTM slave StatefulSet. In this case, the pods will contain an extra container with RedisTM Sentinel. This container will form a cluster of RedisTM Sentinel nodes, which will promote a new master in case the actual one fails. In addition to this, only one service is exposed: + + - RedisTM service: Exposes port 6379 for RedisTM read-only operations and port 26379 for accessing RedisTM Sentinel. + +For read-only operations, access the service using port 6379. For write operations, it's necessary to access the RedisTM Sentinel cluster and query the current master using the command below (using redis-cli or similar: + +``` +SENTINEL get-master-addr-by-name +``` +This command will return the address of the current master, which can be accessed from inside the cluster. + +In case the current master crashes, the Sentinel containers will elect a new master node. + +### Using password file +To use a password file for RedisTM you need to create a secret containing the password. + +> *NOTE*: It is important that the file with the password must be called `redis-password` + +And then deploy the Helm Chart using the secret name as parameter: + +```console +usePassword=true +usePasswordFile=true +existingSecret=redis-password-file +sentinels.enabled=true +metrics.enabled=true +``` + +### Securing traffic using TLS + +TLS support can be enabled in the chart by specifying the `tls.` parameters while creating a release. The following parameters should be configured to properly enable the TLS support in the chart: + +- `tls.enabled`: Enable TLS support. Defaults to `false` +- `tls.certificatesSecret`: Name of the secret that contains the certificates. No defaults. +- `tls.certFilename`: Certificate filename. No defaults. +- `tls.certKeyFilename`: Certificate key filename. No defaults. +- `tls.certCAFilename`: CA Certificate filename. No defaults. + +For example: + +First, create the secret with the certificates files: + +```console +kubectl create secret generic certificates-tls-secret --from-file=./cert.pem --from-file=./cert.key --from-file=./ca.pem +``` + +Then, use the following parameters: + +```console +tls.enabled="true" +tls.certificatesSecret="certificates-tls-secret" +tls.certFilename="cert.pem" +tls.certKeyFilename="cert.key" +tls.certCAFilename="ca.pem" +``` + +### Metrics + +The chart optionally can start a metrics exporter for [prometheus](https://prometheus.io). The metrics endpoint (port 9121) is exposed in the service. Metrics can be scraped from within the cluster using something similar as the described in the [example Prometheus scrape configuration](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml). If metrics are to be scraped from outside the cluster, the Kubernetes API proxy can be utilized to access the endpoint. + +If you have enabled TLS by specifying `tls.enabled=true` you also need to specify TLS option to the metrics exporter. You can do that via `metrics.extraArgs`. You can find the metrics exporter CLI flags for TLS [here](https://github.com/oliver006/redis_exporter#command-line-flags). For example: + +You can either specify `metrics.extraArgs.skip-tls-verification=true` to skip TLS verification or providing the following values under `metrics.extraArgs` for TLS client authentication: + +```console +tls-client-key-file +tls-client-cert-file +tls-ca-cert-file +``` + +### Host Kernel Settings + +RedisTM may require some changes in the kernel of the host machine to work as expected, in particular increasing the `somaxconn` value and disabling transparent huge pages. +To do so, you can set up a privileged initContainer with the `sysctlImage` config values, for example: + +``` +sysctlImage: + enabled: true + mountHostSys: true + command: + - /bin/sh + - -c + - |- + sysctl -w net.core.somaxconn=10000 + echo never > /host-sys/kernel/mm/transparent_hugepage/enabled +``` + +Alternatively, for Kubernetes 1.12+ you can set `securityContext.sysctls` which will configure sysctls for master and slave pods. Example: + +```yaml +securityContext: + sysctls: + - name: net.core.somaxconn + value: "10000" +``` + +Note that this will not disable transparent huge tables. + +## Persistence + +By default, the chart mounts a [Persistent Volume](http://kubernetes.io/docs/user-guide/persistent-volumes/) at the `/data` path. The volume is created using dynamic volume provisioning. If a Persistent Volume Claim already exists, specify it during installation. + +### Existing PersistentVolumeClaim + +1. Create the PersistentVolume +2. Create the PersistentVolumeClaim +3. Install the chart + +```bash +$ helm install my-release --set persistence.existingClaim=PVC_NAME bitnami/redis +``` + +## Backup and restore + +### Backup + +To perform a backup you will need to connect to one of the nodes and execute: + +```bash +$ kubectl exec -it my-redis-master-0 bash + +$ redis-cli +127.0.0.1:6379> auth your_current_redis_password +OK +127.0.0.1:6379> save +OK +``` + +Then you will need to get the created dump file form the redis node: + +```bash +$ kubectl cp my-redis-master-0:/data/dump.rdb dump.rdb -c redis +``` + +### Restore + +To restore in a new cluster, you will need to change a parameter in the redis.conf file and then upload the `dump.rdb` to the volume. + +Follow the following steps: + +- First you will need to set in the `values.yaml` the parameter `appendonly` to `no`, if it is already `no` you can skip this step. + +```yaml +configmap: |- + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly no + # Disable RDB persistence, AOF persistence already enabled. + save "" +``` + +- Start the new cluster to create the PVCs. + +For example, : + +```bash +helm install new-redis -f values.yaml . --set cluster.enabled=true --set cluster.slaveCount=3 +``` + +- Now that the PVC were created, stop it and copy the `dump.rdp` on the persisted data by using a helping pod. + +``` +$ helm delete new-redis + +$ kubectl run --generator=run-pod/v1 -i --rm --tty volpod --overrides=' +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "redisvolpod" + }, + "spec": { + "containers": [{ + "command": [ + "tail", + "-f", + "/dev/null" + ], + "image": "bitnami/minideb", + "name": "mycontainer", + "volumeMounts": [{ + "mountPath": "/mnt", + "name": "redisdata" + }] + }], + "restartPolicy": "Never", + "volumes": [{ + "name": "redisdata", + "persistentVolumeClaim": { + "claimName": "redis-data-new-redis-master-0" + } + }] + } +}' --image="bitnami/minideb" + +$ kubectl cp dump.rdb redisvolpod:/mnt/dump.rdb +$ kubectl delete pod volpod +``` + +- Start again the cluster: + +``` +helm install new-redis -f values.yaml . --set cluster.enabled=true --set cluster.slaveCount=3 +``` + +## NetworkPolicy + +To enable network policy for RedisTM, install +[a networking plugin that implements the Kubernetes NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), +and set `networkPolicy.enabled` to `true`. + +For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting +the DefaultDeny namespace annotation. Note: this will enforce policy for _all_ pods in the namespace: + + kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" + +With NetworkPolicy enabled, only pods with the generated client label will be +able to connect to RedisTM. This label will be displayed in the output +after a successful install. + +With `networkPolicy.ingressNSMatchLabels` pods from other namespaces can connect to redis. Set `networkPolicy.ingressNSPodMatchLabels` to match pod labels in matched namespace. For example, for a namespace labeled `redis=external` and pods in that namespace labeled `redis-client=true` the fields should be set: + +``` +networkPolicy: + enabled: true + ingressNSMatchLabels: + redis: external + ingressNSPodMatchLabels: + redis-client: true +``` + +## Troubleshooting + +Find more information about how to deal with common errors related to Bitnami’s Helm charts in [this troubleshooting guide](https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues). + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 11.0.0 + +When using sentinel, a new statefulset called `-node` was introduced. This will break upgrading from a previous version where the statefulsets are called master and slave. Hence the PVC will not match the new naming and won't be reused. If you want to keep your data, you will need to perform a backup and then a restore the data in this new version. + +### To 10.0.0 + +For releases with `usePassword: true`, the value `sentinel.usePassword` controls whether the password authentication also applies to the sentinel port. This defaults to `true` for a secure configuration, however it is possible to disable to account for the following cases: + +- Using a version of redis-sentinel prior to `5.0.1` where the authentication feature was introduced. +- Where redis clients need to be updated to support sentinel authentication. + +If using a master/slave topology, or with `usePassword: false`, no action is required. + +### To 8.0.18 + +For releases with `metrics.enabled: true` the default tag for the exporter image is now `v1.x.x`. This introduces many changes including metrics names. You'll want to use [this dashboard](https://github.com/oliver006/redis_exporter/blob/master/contrib/grafana_prometheus_redis_dashboard.json) now. Please see the [redis_exporter github page](https://github.com/oliver006/redis_exporter#upgrading-from-0x-to-1x) for more details. + +### To 7.0.0 + +This version causes a change in the RedisTM Master StatefulSet definition, so the command helm upgrade would not work out of the box. As an alternative, one of the following could be done: + +- Recommended: Create a clone of the RedisTM Master PVC (for example, using projects like [this one](https://github.com/edseymour/pvc-transfer)). Then launch a fresh release reusing this cloned PVC. + + ``` + helm install my-release bitnami/redis --set persistence.existingClaim= + ``` + +- Alternative (not recommended, do at your own risk): `helm delete --purge` does not remove the PVC assigned to the RedisTM Master StatefulSet. As a consequence, the following commands can be done to upgrade the release + + ``` + helm delete --purge + helm install bitnami/redis + ``` + +Previous versions of the chart were not using persistence in the slaves, so this upgrade would add it to them. Another important change is that no values are inherited from master to slaves. For example, in 6.0.0 `slaves.readinessProbe.periodSeconds`, if empty, would be set to `master.readinessProbe.periodSeconds`. This approach lacked transparency and was difficult to maintain. From now on, all the slave parameters must be configured just as it is done with the masters. + +Some values have changed as well: + +- `master.port` and `slave.port` have been changed to `redisPort` (same value for both master and slaves) +- `master.securityContext` and `slave.securityContext` have been changed to `securityContext`(same values for both master and slaves) + +By default, the upgrade will not change the cluster topology. In case you want to use RedisTM Sentinel, you must explicitly set `sentinel.enabled` to `true`. + +### To 6.0.0 + +Previous versions of the chart were using an init-container to change the permissions of the volumes. This was done in case the `securityContext` directive in the template was not enough for that (for example, with cephFS). In this new version of the chart, this container is disabled by default (which should not affect most of the deployments). If your installation still requires that init container, execute `helm upgrade` with the `--set volumePermissions.enabled=true`. + +### To 5.0.0 + +The default image in this release may be switched out for any image containing the `redis-server` +and `redis-cli` binaries. If `redis-server` is not the default image ENTRYPOINT, `master.command` +must be specified. + +#### Breaking changes + +- `master.args` and `slave.args` are removed. Use `master.command` or `slave.command` instead in order to override the image entrypoint, or `master.extraFlags` to pass additional flags to `redis-server`. +- `disableCommands` is now interpreted as an array of strings instead of a string of comma separated values. +- `master.persistence.path` now defaults to `/data`. + +### 4.0.0 + +This version removes the `chart` label from the `spec.selector.matchLabels` +which is immutable since `StatefulSet apps/v1beta2`. It has been inadvertently +added, causing any subsequent upgrade to fail. See https://github.com/helm/charts/issues/7726. + +It also fixes https://github.com/helm/charts/issues/7726 where a deployment `extensions/v1beta1` can not be upgraded if `spec.selector` is not explicitly set. + +Finally, it fixes https://github.com/helm/charts/issues/7803 by removing mutable labels in `spec.VolumeClaimTemplate.metadata.labels` so that it is upgradable. + +In order to upgrade, delete the RedisTM StatefulSet before upgrading: + +```bash +kubectl delete statefulsets.apps --cascade=false my-release-redis-master +``` + +And edit the RedisTM slave (and metrics if enabled) deployment: + +```bash +kubectl patch deployments my-release-redis-slave --type=json -p='[{"op": "remove", "path": "/spec/selector/matchLabels/chart"}]' +kubectl patch deployments my-release-redis-metrics --type=json -p='[{"op": "remove", "path": "/spec/selector/matchLabels/chart"}]' +``` + +## Upgrading + +### To 12.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +### To 11.0.0 + +When deployed with sentinel enabled, only a group of nodes is deployed and the master/slave role is handled in the group. To avoid breaking the compatibility, the settings for this nodes are given through the `slave.xxxx` parameters in `values.yaml` + +### To 9.0.0 + +The metrics exporter has been changed from a separate deployment to a sidecar container, due to the latest changes in the RedisTM exporter code. Check the [official page](https://github.com/oliver006/redis_exporter/) for more information. The metrics container image was changed from oliver006/redis_exporter to bitnami/redis-exporter (Bitnami's maintained package of oliver006/redis_exporter). + +### To 7.0.0 + +In order to improve the performance in case of slave failure, we added persistence to the read-only slaves. That means that we moved from Deployment to StatefulSets. This should not affect upgrades from previous versions of the chart, as the deployments did not contain any persistence at all. + +This version also allows enabling RedisTM Sentinel containers inside of the RedisTM Pods (feature disabled by default). In case the master crashes, a new RedisTM node will be elected as master. In order to query the current master (no redis master service is exposed), you need to query first the Sentinel cluster. Find more information [in this section](#master-slave-with-sentinel). diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/.helmignore b/scripts/helm/helm/databases/charts/redis/charts/common/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/Chart.yaml b/scripts/helm/helm/databases/charts/redis/charts/common/Chart.yaml new file mode 100644 index 000000000..bcc3808d0 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.4.2 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- email: containers@bitnami.com + name: Bitnami +name: common +sources: +- https://github.com/bitnami/charts +- http://www.bitnami.com/ +type: library +version: 1.4.2 diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/README.md b/scripts/helm/helm/databases/charts/redis/charts/common/README.md new file mode 100644 index 000000000..559788e39 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/README.md @@ -0,0 +1,322 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 0.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.com/) for deployment and management of Helm Charts in clusters. This Helm chart has been tested on top of [Bitnami Kubernetes Production Runtime](https://kubeprod.io/) (BKPR). Deploy BKPR to get automated TLS certificates, logging and monitoring for your applications. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 3.1.0 + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.node.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.node.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pod.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pod.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|----------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|--------------------------|----------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Return the proper Docker Image Registry Secret Names | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-------------------------|------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.affinities.node.soft` | Return a soft nodeAffinity definition | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for RedisTM are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets. + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 --decode) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 --decode) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_affinities.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_affinities.tpl new file mode 100644 index 000000000..493a6dc7e --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_affinities.tpl @@ -0,0 +1,94 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + namespaces: + - {{ .context.Release.Namespace | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + namespaces: + - {{ .context.Release.Namespace | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_capabilities.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_capabilities.tpl new file mode 100644 index 000000000..4dde56a38 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_capabilities.tpl @@ -0,0 +1,95 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_errors.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_errors.tpl new file mode 100644 index 000000000..a79cc2e32 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_images.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_images.tpl new file mode 100644 index 000000000..60f04fd6e --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_images.tpl @@ -0,0 +1,47 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_ingress.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_ingress.tpl new file mode 100644 index 000000000..622ef50e3 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_ingress.tpl @@ -0,0 +1,42 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if typeIs "int" .servicePort }} + number: {{ .servicePort }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_labels.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_labels.tpl new file mode 100644 index 000000000..252066c7e --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_names.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_names.tpl new file mode 100644 index 000000000..adf2a74f4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_names.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_secrets.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_secrets.tpl new file mode 100644 index 000000000..60b84a701 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_secrets.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- if index $secret.data .key }} + {{- $password = index $secret.data .key }} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_storage.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_storage.tpl new file mode 100644 index 000000000..60e2a844f --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_tplvalues.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_tplvalues.tpl new file mode 100644 index 000000000..2db166851 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_utils.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_utils.tpl new file mode 100644 index 000000000..ea083a249 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 --decode) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_warnings.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_warnings.tpl new file mode 100644 index 000000000..ae10fa41e --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 000000000..8679ddffb --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 000000000..bb5ed7253 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 000000000..7d5ecbccb --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB(R) required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB(R) values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB(R) is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (not $existingSecret) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB(R) is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 000000000..992bcd390 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,131 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + + {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_redis.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_redis.tpl new file mode 100644 index 000000000..3e2a47c03 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,72 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis(TM) required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $existingSecret := include "common.redis.values.existingSecret" . -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $valueKeyRedisPassword := printf "%s%s" $valueKeyPrefix "password" -}} + {{- $valueKeyRedisUsePassword := printf "%s%s" $valueKeyPrefix "usePassword" -}} + + {{- if and (not $existingSecret) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $usePassword := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUsePassword "context" .context) -}} + {{- if eq $usePassword "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Redis Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.redis.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Redis(TM) is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.redis.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_validations.tpl b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_validations.tpl new file mode 100644 index 000000000..9a814cf40 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/values.yaml b/scripts/helm/helm/databases/charts/redis/charts/common/values.yaml new file mode 100644 index 000000000..9ecdc93f5 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/charts/common/values.yaml @@ -0,0 +1,3 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +exampleValue: common-chart diff --git a/scripts/helm/helm/databases/charts/redis/ci/default-values.yaml b/scripts/helm/helm/databases/charts/redis/ci/default-values.yaml new file mode 100644 index 000000000..fc2ba605a --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/ci/default-values.yaml @@ -0,0 +1 @@ +# Leave this file empty to ensure that CI runs builds against the default configuration in values.yaml. diff --git a/scripts/helm/helm/databases/charts/redis/ci/extra-flags-values.yaml b/scripts/helm/helm/databases/charts/redis/ci/extra-flags-values.yaml new file mode 100644 index 000000000..71132f76e --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/ci/extra-flags-values.yaml @@ -0,0 +1,11 @@ +master: + extraFlags: + - --maxmemory-policy allkeys-lru + persistence: + enabled: false +slave: + extraFlags: + - --maxmemory-policy allkeys-lru + persistence: + enabled: false +usePassword: false diff --git a/scripts/helm/helm/databases/charts/redis/ci/production-sentinel-values.yaml b/scripts/helm/helm/databases/charts/redis/ci/production-sentinel-values.yaml new file mode 100644 index 000000000..009a3718a --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/ci/production-sentinel-values.yaml @@ -0,0 +1,682 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + # imageRegistry: myRegistryName + # imagePullSecrets: + # - myRegistryKeySecretName + # storageClass: myStorageClass + redis: {} + +## Bitnami Redis(TM) image version +## ref: https://hub.docker.com/r/bitnami/redis/tags/ +## +image: + registry: docker.io + repository: bitnami/redis + ## Bitnami Redis(TM) image tag + ## ref: https://github.com/bitnami/bitnami-docker-redis#supported-tags-and-respective-dockerfile-links + ## + tag: 5.0.9-debian-10-r0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + +## String to partially override redis.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override redis.fullname template +## +# fullnameOverride: + +## Cluster settings +cluster: + enabled: true + slaveCount: 3 + +## Use redis sentinel in the redis pod. This will disable the master and slave services and +## create one redis service with ports to the sentinel and the redis instances +sentinel: + enabled: true + ## Require password authentication on the sentinel itself + ## ref: https://redis.io/topics/sentinel + usePassword: true + ## Bitnami Redis(TM) Sentintel image version + ## ref: https://hub.docker.com/r/bitnami/redis-sentinel/tags/ + ## + image: + registry: docker.io + repository: bitnami/redis-sentinel + ## Bitnami Redis(TM) image tag + ## ref: https://github.com/bitnami/bitnami-docker-redis-sentinel#supported-tags-and-respective-dockerfile-links + ## + tag: 5.0.9-debian-10-r0 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + masterSet: mymaster + initialCheckTimeout: 5 + quorum: 2 + downAfterMilliseconds: 60000 + failoverTimeout: 18000 + parallelSyncs: 1 + port: 26379 + ## Additional Redis(TM) configuration for the sentinel nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Enable or disable static sentinel IDs for each replicas + ## If disabled each sentinel will generate a random id at startup + ## If enabled, each replicas will have a constant ID on each start-up + ## + staticID: false + ## Configure extra options for Redis(TM) Sentinel liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + customLivenessProbe: {} + customReadinessProbe: {} + ## Redis(TM) Sentinel resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + ## Redis(TM) Sentinel Service properties + service: + ## Redis(TM) Sentinel Service type + type: ClusterIP + sentinelPort: 26379 + redisPort: 6379 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # sentinelNodePort: + # redisNodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + loadBalancerIP: + +## Specifies the Kubernetes Cluster's Domain Name. +## +clusterDomain: cluster.local + +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: true + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis(TM) is listening + ## on. When true, Redis(TM) will accept connections from any source + ## (with the correct destination port). + ## + # allowExternal: true + + ## Allow connections from other namespacess. Just set label for namespace and set label for pods (optional). + ## + ingressNSMatchLabels: {} + ingressNSPodMatchLabels: {} + +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fullname template + name: + +rbac: + ## Specifies whether RBAC resources should be created + ## + create: false + + role: + ## Rules to create. It follows the role specification + # rules: + # - apiGroups: + # - extensions + # resources: + # - podsecuritypolicies + # verbs: + # - use + # resourceNames: + # - gce.unprivileged + rules: [] + +## Redis(TM) pod Security Context +securityContext: + enabled: true + fsGroup: 1001 + runAsUser: 1001 + ## sysctl settings for master and slave pods + ## + ## Uncomment the setting below to increase the net.core.somaxconn value + ## + # sysctls: + # - name: net.core.somaxconn + # value: "10000" + +## Use password authentication +usePassword: true +## Redis(TM) password (both master and slave) +## Defaults to a random 10-character alphanumeric string if not set and usePassword is true +## ref: https://github.com/bitnami/bitnami-docker-redis#setting-the-server-password-on-first-run +## +password: +## Use existing secret (ignores previous password) +# existingSecret: +## Password key to be retrieved from Redis(TM) secret +## +# existingSecretPasswordKey: + +## Mount secrets as files instead of environment variables +usePasswordFile: false + +## Persist data to a persistent volume (Redis(TM) Master) +persistence: + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + existingClaim: + +# Redis(TM) port +redisPort: 6379 + +## +## Redis(TM) Master parameters +## +master: + ## Redis(TM) command arguments + ## + ## Can be used to specify command line arguments, for example: + ## + command: "/run.sh" + ## Additional Redis(TM) configuration for the master nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Redis(TM) additional command line flags + ## + ## Can be used to specify command line flags, for example: + ## + ## extraFlags: + ## - "--maxmemory-policy volatile-ttl" + ## - "--repl-backlog-size 1024mb" + extraFlags: [] + ## Comma-separated list of Redis(TM) commands to disable + ## + ## Can be used to disable Redis(TM) commands for security reasons. + ## Commands will be completely disabled by renaming each to an empty string. + ## ref: https://redis.io/topics/security#disabling-of-specific-commands + ## + disableCommands: + - FLUSHDB + - FLUSHALL + + ## Redis(TM) Master additional pod labels and annotations + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + podLabels: {} + podAnnotations: {} + + ## Redis(TM) Master resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: + + ## Configure extra options for Redis(TM) Master liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + + ## Configure custom probes for images other images like + ## rhscl/redis-32-rhel7 rhscl/redis-5-rhel7 + ## Only used if readinessProbe.enabled: false / livenessProbe.enabled: false + ## + # customLivenessProbe: + # tcpSocket: + # port: 6379 + # initialDelaySeconds: 10 + # periodSeconds: 5 + # customReadinessProbe: + # initialDelaySeconds: 30 + # periodSeconds: 10 + # timeoutSeconds: 5 + # exec: + # command: + # - "container-entrypoint" + # - "bash" + # - "-c" + # - "redis-cli set liveness-probe \"`date`\" | grep OK" + customLivenessProbe: {} + customReadinessProbe: {} + + ## Redis(TM) Master Node selectors and tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + ## Redis(TM) Master pod/node affinity/anti-affinity + ## + affinity: {} + + ## Redis(TM) Master Service properties + service: + ## Redis(TM) Master Service type + type: ClusterIP + port: 6379 + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + loadBalancerIP: + # loadBalancerSourceRanges: ["10.0.0.0/8"] + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + ## The path the volume will be mounted at, useful when using different + ## Redis(TM) images. + path: /data + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + subPath: "" + ## redis data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + ## Persistent Volume selectors + ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#selector + matchLabels: {} + matchExpressions: {} + + ## Update strategy, can be set to RollingUpdate or onDelete by default. + ## https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + statefulset: + updateStrategy: RollingUpdate + ## Partition update strategy + ## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions + # rollingUpdatePartition: + + ## Redis(TM) Master pod priorityClassName + ## + priorityClassName: {} + +## +## Redis(TM) Slave properties +## Note: service.type is a mandatory parameter +## The rest of the parameters are either optional or, if undefined, will inherit those declared in Redis(TM) Master +## +slave: + ## Slave Service properties + service: + ## Redis(TM) Slave Service type + type: ClusterIP + ## Redis(TM) port + port: 6379 + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + loadBalancerIP: + # loadBalancerSourceRanges: ["10.0.0.0/8"] + + ## Redis(TM) slave port + port: 6379 + ## Can be used to specify command line arguments, for example: + ## + command: "/run.sh" + ## Additional Redis(TM) configuration for the slave nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Redis(TM) extra flags + extraFlags: [] + ## List of Redis(TM) commands to disable + disableCommands: + - FLUSHDB + - FLUSHALL + + ## Redis(TM) Slave pod/node affinity/anti-affinity + ## + affinity: {} + + ## Configure extra options for Redis(TM) Slave liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + + ## Configure custom probes for images other images like + ## rhscl/redis-32-rhel7 rhscl/redis-5-rhel7 + ## Only used if readinessProbe.enabled: false / livenessProbe.enabled: false + ## + # customLivenessProbe: + # tcpSocket: + # port: 6379 + # initialDelaySeconds: 10 + # periodSeconds: 5 + # customReadinessProbe: + # initialDelaySeconds: 30 + # periodSeconds: 10 + # timeoutSeconds: 5 + # exec: + # command: + # - "container-entrypoint" + # - "bash" + # - "-c" + # - "redis-cli set liveness-probe \"`date`\" | grep OK" + customLivenessProbe: {} + customReadinessProbe: {} + + ## Redis(TM) slave Resource + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + + ## Redis(TM) slave selectors and tolerations for pod assignment + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: + + ## Redis(TM) slave pod Annotation and Labels + podLabels: {} + podAnnotations: {} + + ## Redis(TM) slave pod priorityClassName + # priorityClassName: {} + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + ## The path the volume will be mounted at, useful when using different + ## Redis(TM) images. + path: /data + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + subPath: "" + ## redis data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + ## Persistent Volume selectors + ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#selector + matchLabels: {} + matchExpressions: {} + + ## Update strategy, can be set to RollingUpdate or onDelete by default. + ## https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + statefulset: + updateStrategy: RollingUpdate + ## Partition update strategy + ## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions + # rollingUpdatePartition: + +## Prometheus Exporter / Metrics +## +metrics: + enabled: true + + image: + registry: docker.io + repository: bitnami/redis-exporter + tag: 1.5.3-debian-10-r14 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Metrics exporter resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + # resources: {} + + ## Extra arguments for Metrics exporter, for example: + ## extraArgs: + ## check-keys: myKey,myOtherKey + # extraArgs: {} + + ## Metrics exporter pod Annotation and Labels + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + # podLabels: {} + + # Enable this if you're using https://github.com/coreos/prometheus-operator + serviceMonitor: + enabled: false + ## Specify a namespace if needed + # namespace: monitoring + # fallback to the prometheus default unless specified + # interval: 10s + ## Defaults to what's used if you follow CoreOS [Prometheus Install Instructions](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#tldr) + ## [Prometheus Selector Label](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#exporters) + selector: + prometheus: kube-prometheus + + ## Custom PrometheusRule to be defined + ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart + ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions + prometheusRule: + enabled: false + additionalLabels: {} + namespace: "" + ## Redis(TM) prometheus rules + ## These are just examples rules, please adapt them to your needs. + ## Make sure to constraint the rules to the current postgresql service. + # rules: + # - alert: RedisDown + # expr: redis_up{service="{{ template "redis.fullname" . }}-metrics"} == 0 + # for: 2m + # labels: + # severity: error + # annotations: + # summary: Redis(TM) instance {{ "{{ $labels.instance }}" }} down + # description: Redis(TM) instance {{ "{{ $labels.instance }}" }} is down + # - alert: RedisMemoryHigh + # expr: > + # redis_memory_used_bytes{service="{{ template "redis.fullname" . }}-metrics"} * 100 + # / + # redis_memory_max_bytes{service="{{ template "redis.fullname" . }}-metrics"} + # > 90 + # for: 2m + # labels: + # severity: error + # annotations: + # summary: Redis(TM) instance {{ "{{ $labels.instance }}" }} is using too much memory + # description: | + # Redis(TM) instance {{ "{{ $labels.instance }}" }} is using {{ "{{ $value }}" }}% of its available memory. + # - alert: RedisKeyEviction + # expr: | + # increase(redis_evicted_keys_total{service="{{ template "redis.fullname" . }}-metrics"}[5m]) > 0 + # for: 1s + # labels: + # severity: error + # annotations: + # summary: Redis(TM) instance {{ "{{ $labels.instance }}" }} has evicted keys + # description: | + # Redis(TM) instance {{ "{{ $labels.instance }}" }} has evicted {{ "{{ $value }}" }} keys in the last 5 minutes. + rules: [] + + ## Metrics exporter pod priorityClassName + # priorityClassName: {} + service: + type: ClusterIP + ## Use serviceLoadBalancerIP to request a specific static IP, + ## otherwise leave blank + # loadBalancerIP: + annotations: {} + labels: {} + +## +## Init containers parameters: +## volumePermissions: Change the owner of the persist volume mountpoint to RunAsUser:fsGroup +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/bitnami-shell + tag: "10" + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + +## Redis(TM) config file +## ref: https://redis.io/topics/config +## +configmap: |- + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly yes + # Disable RDB persistence, AOF persistence already enabled. + save "" + +## Sysctl InitContainer +## used to perform sysctl operation to modify Kernel settings (needed sometimes to avoid warnings) +sysctlImage: + enabled: false + command: [] + registry: docker.io + repository: bitnami/bitnami-shell + tag: "10" + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + mountHostSys: false + resources: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + +## PodSecurityPolicy configuration +## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + ## Specifies whether a PodSecurityPolicy should be created + ## + create: false diff --git a/scripts/helm/helm/databases/charts/redis/img/redis-cluster-topology.png b/scripts/helm/helm/databases/charts/redis/img/redis-cluster-topology.png new file mode 100644 index 0000000000000000000000000000000000000000..f0a02a9f8835381302731c9cb000b2835a45e7c9 GIT binary patch literal 11448 zcmeI2cUx22x9_8Zh@yyC0I4cPR3wC|kVp$H5PB1ZB#;m~gf1}_1O)^|QB*Y2#4gyt zMlbeAQxFvp712m>hX{m(GZyaOx!ZkSz`gf*j(>!>GS^(QjWNFCGu9zzC!56!6&9jU zsKs`+R<0=2Tv-%qj{Ji8aAni0vR)KQEHl>HJ2pI#N)HP{sbegEe^b}f4US~Qs$;Cw z_4G(lQ96Ni5-o-l&d`YniiJz?dw66Zok|Z1{M|-RS5J47uKp%8#vQGzjxpCah7XLM zj-j5O@9*{`T2RE_9UAE9LI+xoBnmwuHj)v%{&$O@SQ71bZ+Kl2DH#*!W~guJ6YNcK zHZTeO`>F9kF${WS#P4QkJslGrH2U}5u}M)uzb^*{#nUN4$W@Fr%;@i-!xQO$57ytD z8g7NACsAo=gGf@4U2vq4|L;yBNa25X;tb>6G}|@C+Q2i|iEP41uyWQ#JBJ%4#8?F< zhJ?qHVxj_RUKr=dpkNn9ac?y zjjf4Zb6GNz?1!C-z2!T|`I6FNrJxgzEdNh&X5<(5KrG*81T6sFq zW4#j0(FP$=R0ktuGQKhJ>5aE1x<}i)7{Yf-uoVV+b&Uyx|9F@?Q)9v%9i!kom8xr_ zZyDi8quLlJ$HoMDMv5Etu;N25ccDbf0|Q5Hey zuy9hmyN{lmxi_9+6lmk*8e~ZKun3K`af^yY(`;?=K0&@@N^D4|ZmdgqQerIL%hrxy zM`KVuoedc@Cs*&NU@z}TC!YjoOLVwfgp0XBq^=QOH`LfZDk)yy$}P!&oMd9pNV2eV za19GKbTn|V3?&(oC>TsM-pj)_-XqzO!f>`XcTI{5(F-#P4|eb+*@i_q#RWz9Cb}lU zY#Dly5oBMwdxVb>J|ZxR=Hn2B(esIp*C%@g#keGsV#(gFfsDuqU)z`@qaaF%b3~+m zaCivCJ1!{#Z-OS0%&mNabUj@xDKwK12OC|CUMSr<(LBO6#=*zKmh2FMcXPB0wAZ(G zOLT#y3^MdVd+FN4H&{DMs+$|$6}}_mun)3ByGD6M!*@NGc#37RH*(}cY~w5(g7s}^ zzJ{deP$OjHtPIHRcv})F*daVJ9OLS0Leg`Niu1-BQNyAVy)cG$-t^cQdT5*_#!w#< z6bFATlk}s>LCIl6HxqO;(KpT^F4{I2o&%lvSS5MJQIi?*1e?StBg+V5BGsE18|WMZ zW@PMb;;rjJh)Z(TCxklSlZ>ctaW=Yo)I?_w8+1I}9FDPYfEk{Rp z-ILpbFFK4J$#eW+T6n?W^l5qJZKu}h5eQ&HLB6NEPly?kFMd+9B*;21U%BtZEs%?pK-aOJA&_a1j0}?4#_1KVkJ~ zb1N$=!~T5LHEY(Ki6ShOkvZyUtN}kg)=-p%p8{pGCE=%=k}YB~GBVw%{}0_J=Dl%c zNm&_1!2faZ@ZlzI_bqfh*IP+9=vgI}rUzo0%pyLDCQ>#KFyN?VT{J-WBp zCg)?vCAY`Vo>`JeY8zKB+H+FMc}Q(BfZHZ_r67MSRsc_F7C zywt+dvh}JDW@2iJ{P>7kKAJK=H#hgl*|VFbw_3cM9SuWCQ8xa>xpUb;&rXoK%1McR zdnJ}CDoP!Dk~;a<;97Nc@fy+86}OQmPoC7l)OA^+lzf7Q)M`ViYrc0ZqM~%M)1#Jw zfq};3$}t=HHo0Bw?;rEaU97FebB`UnQf>OC*hZRgcFPL?lreG9^k2lGzd~ZiK3a#p zy)Aj;;m?Kl6GpPH?|86AUw`=-!|+_};cth8N7wOf@{BbUz8)iD>yD^dMsdu3sjgcm zW@l$t760Qj&u4MW_&1NM{x!;Vb#<@w7h9$;X)0sy)@H8k+&6o3!pHoJ>1>urexUjA z{*xzH?Mxn059_{ZJ+&^q>tPZG7Orn-Nb**9*6rK+L&p|ya{?=HmlwBKDP8rsHUCla zcztmWHlwP_c!*J;ZD_dCY!|w?1!4%~4*Dqs@kWrCfWZ=WDk*3@9_9&Rg z5F_^U)4UCdKkmGFRb;&gdi0)Bv?Q|{S_2<>6TK>ZjGG z9{XfuWax&!)fbyTe+?S#0yhM(;#x%99AT%DK(HZO64jEUVl(pL5m;Fr&fwZPdwct( z%F5ZbwcE;A()V1-YtksJyA;;q7Ewg>BpOX=mYSHHB=uG^+sd@U;l8gOmt?7&bDNkM zu3c;-9?ml75dN@2UC14Kwt46Ayu3oYV;XHVV{t*d3K&7wJ2t7s>ig?!V=XEy zTei8u;}z>#O<*5YkYo515Ix`}TT?sxu@^5d`*D}*ybg@r&so3e+v-a$5 zZUo2RH_Wjx%ILh}MVX!$Yom3g3NIn!T+5S?3nT>6XvGdEb=qi07N;DkD{zvz0b5=XcjZ z%dB}4OKjOLo4Fig7=&v@&9~z!vfyUH!r>W8>ohdhn40=cWhIkX&B#nMgdSzQe698C z&HwuHqLib$HKeUX^6As3C7g@0e3~gTl8)Z9$CYK#Xmq|lJ(njRXfx1~)AFtJO6k9w zkVqt3A`$kLt7TxY0_>xW)*>D8sh7e;aXnntiCP%~e12HDO9BW)FCB@FA<~X*CewMZ8<>mEH4zkm8a-`u%DeyvRRBF-r^R>#fZ%=tc?G*uabB8b?O`
    gt0u3YK+3VbZ)7Cr=g*&)YecY;+uHRLladT3#L-cG+Z4CN zKHgdOtr;9@>`b#Ba+WJ_{4yh$6P!auoRD{nC(pX_v#ErxkgbB-IsWFP&W_}s%qJmt z*68S5|2A?sI~z}@uitaVzm50v=e`T{#J^q`x%l~|^x)X1PD%ECQ3)6r7}&pZlW#!x z&7i92f%*0h4r6aaZ}j74CZ$tSQjUNV{3&o&kyxTw>qazFXX3Act31{S^Z5WvqP%;8 zRA3x!@p~>={SD~@*D62Me{9N}gIbv>4iVJ*7S*vcbLjhxnzRda8yg$XpFfY95st75=?aFC9}wq0 zF)@LdjmM0RV|#f>r?%h2M3;$#Pa{fO)5_jR(9T%&W!y0{L3|lv0qAhQ-1KmF^PQ#ZP>O;1`%Dz!7D2~MQU ziWTB2Dk_SwVr4}3^O^Dya_Y^8?nk&h01c(e<# zn#xC@6l4*7aQx)S269E;yL${|OE<2}jM=QQ;)KaRrkkog^P>D`le>${Uf8$b)1Kp> zs|yMWF0X0_-=xd4u&0Y3+CqaAK$nqa%A?fkq#lyJoui-Hen2>X}amW4T zsHl&Pl3-Q+5SF_hB#eanpW7uIe*?#A_w<-e)@q~SyPJD|z&6%iOR!QgO3a%bavc#AD>^6l_mA0H!azv|NZlSo-Y>{7yA@;9CP`qRRbUn%6+e)_JqE@(-#Tj($SLZ+CP~bsy z^z=kD2pv1Nx#8&8SRE{FDJOU-zYL2!Y4}F2TC*4mHh4*NbaWfddEQmOd&^+c{tSn~ z1k|3gva*{asc|0Qw8G!sK8kJ7>0&1%@zj--rXaiP*RVBWzVhzUM4V8cKhnFInYN=G zH(rj*4J;8!8EBy5Shg7(^gc>&AhW&&K%fp5%101T+q}K=U2!!ZR+L`2pe;1(yhCD5 zwZLA;2r!@WQK-ns$WLFsz)Bk!GrmY|iT$d|OtY-O3CYmGMWK8=bZ|_=(b>~egE`;B z&DC`utbzl(e>s=PLdF(@$Qo;F>;K4_clR(EnVF@3${I-&7|nq*XIAU@RjD#F9Ja!- z1gAUx{SAmEZGM?Fa-d@lh@@~DD8%M>?%ZkX;BXAvqz+>)+VLR1jdqa4x`}xGPujwJ zjrkWlZqbNgA0GNlNuA6sD9GCRDA|r(xQNS{?CRYFpMIBcClzE-x)A0>$IUe54MLV{ z5YKI7rd=<`4U$-_CQ(64=M>`}9L2G*vBc3jiKCFeFf-JY%#xS?czdS-j34BuI)Bic zdm|o%rQaRgNkv&y6rbx8JJcqgxpGr3w#j*=j^6?ni&d~S!D(vw4hk!~PV_BJXp*+! z*WgV`k!x(MK7_Aw8b^7^?KuC1QSW&W@?d1=cj}_1HTsUh>3@C-(>ADeNNAuaxTPcrHRdNQYOyka!)+4~pt_Se?%`LR z;xnbVV}q7fEhKKOH~ewy)Ya|Slm*-y!(D7)Ma_K*%o**nUm81s`t?3R<$}}M49N5# zO;E?-RNO8Q>V&^~b0AxTn{2rwaMM6V%>xSL3uu>2xLgQhlQT=!U!rWP1^I{yYZ~lH zCiu;Dr4yU|b(q->0jx?`SqgmC)WAIw&;orf@uOty$OokjUf03z=V70$*Vfi9CpRq= z6BFxuqRFa7_Xe7xHZ?Q6iq)$~_dm)_vy2nS?9xW&ct6KH?@M@!S2w7Qj z=(&&|);AuKJ$mZYdgjW#!lRHK-s#T|KF`4lq}0^Zf;%qCq)Ex|YizU%>b>pm?R_vc zRSXGr(F4tL*c4;KurAeqW_0EIH*LSGYHF$oXUzJ|^&Mz>+qx9d`YfxIVcpv04@u z7P_>TibzHZ0gUm@th>A0*2$^4z=W}M*)l}U-VX0sxfY*Qo3UcUk-WUm9lIV^fIX2} z*E1>gbxsAwdp~^&w6d{TYiukBsv-@H2jbq;uaTU`k8PiyHdu7U^IHAuYulqIMoAFE z)@W)<+`D%Vl!lF+oh0gfNr@P^e{hY~PEJeofU`lfu=n>@z+N=MaRRq@2)rNV)8nJT z@2b%EV~6$~IkFUaoQa9NiHS*l&Du`{aQhXwH+5(=#3Ug)*1bHtRYpZ6C+V**r214z zInzX4$S#&HU5fJe_fOoltD&J`-itFv$e}!K6uCrNS{k|d=FO@(bLK2qw5Xx4Za)P6 z0|_6JB*5sp?lCk$heZuEFCHT1#U4bGY#KpPcA?gW4GpmOE{Mljcuy6aXct?-SY!Dp z3!^+du-4Tf9pP_A2VlmeeOgNzIH%HHIGOnLW(iBYVtfv6^)iTckbN8)b|_e zoVEKYtaXQkm(nQ~GAVfY2Jx!TxY1AzKg z4W8SD1hSHnb$$=x9jC6&zn46&{rZM~PG4U@@aeQd%k+=!L|h&=GXz-6GC}dyXhKH= z2RD-ik+|;JvqhU9@s`~U>t4gmwFDg}4-1F*+3qmBukUG|ofzK+)Lv&ArD)0Pt3LZd z-u1WIs95{?74S@)R;*r~3HkcRiWX(dl$3`&lRqTHrj?s_W*^34BcsF5WXY98Q$y+qe39R*cE(Z(E%W;ALmX4_goK$fL!PTdlU!K_L zGZ#S${GQ0RvabpPh7{k+h|59y-9z+iMMVWtafpeDDIuauU^oMX>B{wQ?}V2ATkEct z1-3xFh3IQt>T@|OYirQg8@QaLu3rBN++g$_kzWV_bX-8s-S9SA)$!xUm-D{1T24vr z2w=Sg{!3!L9JEK2I^=1iqodoX)C;`zI_trjLECF>-h2{kEhacNi=@1UIHQz$az!l| z8)%lcvZ~uaCu?84c=5SCkAqGC7kmLkoDB}^lYD!N{7&jB&RM& zLL_Grvo2fFWDTItH85H@m+%ZR2)7Vtv%1**&5dUws}CJmvmB zGtS1^z4OMP90a2@w6(J{cXmF)lV1!SAerA&p@-{k0=K79n$nUDi8{9rS~df41@yU! zPg8_JTD8h3G71pC{{UNtA{1TRzuwRQ1S%=35%6pHJ^Yjj1%R3w-0SN*GK-4VKt?7H z+!zW;Nl8h@(I3T-s)4h!D)aJ1NHvB;hl3ET%cg*faQRJkm08o#)4X;fbmidWztg>;!#64Q4Gor0j7EW=${<=D*66b5 zujA|8puV!ZCvyl$p>fKDE~*bq0LeZf$BO##)&K#p0HJ_%O}wcQ9Y9h)n8zMKySw-s zXZgX*^dPPxyvOhfvCXxg%sc?h3$;R^(_K(P3*UU9 zmo=l*kS={AxsHB14URAhG(H6XjehrIFNH9@s`H1Z0V^NwC<~Q+g!6yY>m|$gN=Rl^ zC)eE2fUMg+rTS^MrFP;Z-c2n#cY!_uVPeiZze#uwRB7j-`Ucr7 z1d%86&I~B~qfip>kaqztb`AD8{~L0y=zSrD@@I~B`d2Tngq+?T{yoYEXi*LkuBUpH z0H|_0bQSw`DM}TKRR+*p$t3cnann&fH_kwO9H+mvI*@?5h2v`?1mF>%Lx?i#e=%R@ zUH7lahQrdIgA+lKduYc@^~aB7Sa?aW0Ti5oJ6nWrkqTQAxuWOM%e$d(bO`56eWtQ} zAwti*<#NWldf&6c`I?qc=!pDa3fZy)Qd5EY%4(=&UH>3%Ritcc}65Wn3V;U zGj2e~MHl15QJ{w7^`fl(h_e)2ahe3|wfW)bzFeGAc0@N6uhW(&X@lDYn22!P7*?JX zS_5i1OF&7$Q0K$m87KWL`6GbnHRhxCfFD-V(Sg!n$ez3nUTd_q?Wd&9l;h%{m%vmJ zYTjW<6sS6P{@L>DPstVMK}`bn6c@P7#^Y}sX9^jr!V9FNkWys-P#L=!m_#Y3d=bA4 zshnkXunU{T&JRo5MO)*iN!DDV5)vM-K@C$91wg?Pk52;}n%U9eURqOj-w@Jegh-NE zX+Y}we*8ELCNYG$IG^^iz#NJ$PzKfFm-!=I+`%_M==-;pa)8z>^u1K?EVY%s@=l- zy!B2{l-)a99)1PoVKEd^t^nOjr_({?$m#Ja;%i|Y6XALlsj?* z>f2SmNb!5t=79F8f=nm{tq1iagrj;_Mx5oXzuzcQ%oKKJ3r$X3;Sc6QhcC~WWrCv& zXf3jCE0cQ)RgQir2!ajTv5P`MK6d%PFUPe+eHQt}6WlYhv(uo|$me#iWoEQhPkvbq zRiQcH$&UT0Oko%H9MdEgMfO%NSiHCiiXkN&7kd{M8Aw*3GGGEM3_qo)KC5{Der=#X z{~{HX6C8=gD>te$AQ(BYi$kGmp1&nd8(e8RqDB}}dyzL1z;k#dz?_lSAL=Aj`lBCJ~vZaRd#_rNgaC%{d^a1(O*c4V(IEKPIWeu+|! z%s41R;Cpis01Q##f4*n&tq)o>%MwlZf?K7)8x|0Q+B8i-h>9Z>i#>Z#rjwtJ+1lD3 z!9G!kDkObY&CdG<*>PCB)+SAnpzcWopr+1YmyVP=wr>GG+}yaX!FnYg^j|U!I@Z5A*1U%+~3V zCPXPnD)TY4C4fAUv>D!~AU7VA@vb0?g9P~_1vT!1n$8ix_3$2(n1nwesZB`Vk~c;^CATva2ZRgf1zp?(eu{DL2oO*VOmYB7 zBSIoS4h$fnf{?E}XTO9r-rHMl?%cV3-@h+{@KL~<7Y!45Ic+v z^)Lto6Ar$AW@U!Y=x^d~2*j3yWE7t4A3^l?BOoM{bpL#lP?Yxy3?WM>=}9Omx&{Tw zdU(5f2D|!)$OaI|a0%}F2YPrDy$K$F+9=8^${&_jKCGaGRZx&n(pFN24@E^;byWqs zKkZ#T2?2ixRFRd30S=kEx_bwZ14Fzd|Fj^GT|NJdW~Fl2&%!mz$V#6`a8_3jwp5Pr z{?jIm5FFwi81Sc=0*o&UkNX`DIWmaw=duSO%-fy7xT=JLH~JeoJkk4au+G{41$dfjkSY4t(D}p^=!R^ z)U|CKrYmJoGw(?iivqLLdoBFF*ll?<2 z{d}S{4I;xL&BN8)(F%r&3f=@&csYzVH1I;Yo0Ig+@#@xYm`Gbo z6;m~mpLZnL*4@I=+EvXe#N5m`(AL<@!_3GsBE%yyGAcA&UeVCaP}fffP1ZH`@bDwr zhD2D}kpm6o6;)Mze6)RWD3Yg@kB^c!$`GxJM4>$4F##cp{%-QFkw)lnTQgk~MI&8P z9Tk10zyNPEcN7YiN#By}ZyI8$7_3L|v~!E_G_ke}4lu`u==l2t+o5oZCgwN=4ABVf zui&QcjmD~akyP<08^&v>n44=yl98cgJ8fdPxxR9gqDi2sKN3SAn}o{~-Th!}D@b7- zBK+@V8i5Z|Ggr5W#Ca+Q!5y+WQr{-X&)36BG1AnE5r%G*B_;-qhuCJ z3XxYc5B0Jokc_ z1Ak8sy+8wFWt6*~nwd6M)r1rorlJ;Lr9cdJSMxSgQB?OfRkKvnLmDd)4PlJ%D3S^z zavWp#D8Z=u|LmCG+Xw#siwR1ok~bZf5C~C(k)Ad-qG*2D7mHoyXI?BnJ}PN*Kx35p zB0Edh32l>qHgUl2g57l&%5g8&D_%rGS=zPh21n3VO3m93Nq4b_7)maAO;F`;?+;dI z-ItZmu3&nclXqOPbM=E;&Mi0Mw-@mX$2tScS4HKOIzK$%kW*?FHp&rSe6q*<=x6aE zqfMNg>J+N6p7{PeDbDP4JY7#u(T%p7vpxBL@S=FafS-O?m}=Pk;F*MkEia0-M7p}V zc-4c&*_k(Q+Pt~zwx>CH=JnS3p_-(c8nsiWPJNi4&-gY!bOoI+J3VmwAhdCn=amm0 zH+=HsiKwjX&TZSa_22d72@Vdnu(y|9yOvYvO3Uev{k8tx(8%cV+qw`=LBVvVhN#Cw zLjofsBfQ6=l+*S1@0E2(-XWms^8M5B)An}lxHwI-iyCKMzs6v(*hW9q&9zRg+}zxU z4j*ozS(RlK7iSg~34VOnAdr%hg0FI6f)VsHg)^$E+^rb`NfeiQbaZvIb8=46=~7+QAA^@?ael-1x%chcmnFW9`5+-`|sGz2%raXVf(`V0i3#DLBQH>AZa&2a2=T~O^{ytu7 z_ntkdHWCdMm|YOL>og08J_eIBjl3EpgIlwDo-b!G)p3ziNGKycoxO{cpb_=;P-$uD ztGW=+n+|mtvLNPAp49N0p}>*sdv4gE+APmW+f;m@<~!-shs{^TtOj?H z3|;Z`HzvWsiitagQf-_fQFqNX>Wm2l31a7^jLVlr{RVEKU_915DJBHl_U#HDPj~6* z>wl@t^21tLolZ_>-LiFS-A|R;t4C#UO^_=i3k#WFzmmHwvGX4upPufy!86%eET_S? z?FlB-z0B<5sp4XhVsiFCD!23pXetu_(~``!`vi6N+V0)EUqSA2@$jTkN{!B(Ia51V zQC~$G)9 zYeYmwrckuanlL?8u(o4U#qoQEP6w1;y}D)ie&`;d(?ut!bjv*l&7OQ7YeThN*2=zk zF=KHts7t|+D1+M(wcKMT8s+r`Ah^x8yOs<60_c?`zhDLm8KUmT?H?Qdx5)dH!?D_MFWZI#S$rq1v zB+wAf5s2Dv@Q!zXOEuAN&0x^rX?S5=J_%t|iN3u&ey6>3h?*0=V^7W~8OvepuYS zvHk<{P1I<&Q|vMhYh$B!B)(gK|5{m@Ze`yU$NDe^tnu^n1LmgXJ4J2#wb~hfxo<8z zJA2>w50|v&B#yC>B$0n!H2PB=3kroQb^j@GwmkvTY3y#FQ}dUJ=S&XqN==Q8$Nl=N z9^28KV?J*cH)DPl{V#bM3xnx-yaP3}NgPM4mWxYSY)bJ`M(cy1MRT<`7O_ zS&2B8pFj5K*SZ{Z1c=Sb&E4AIlqrGE!&jnn#BbWy?BtPi;`%AH>U+?v==j3B21pOa zB;M3b+DVT{dILn$>>~eRy6ei~_wNR=vB$oA`NG)Rzt$Jxdojmi*OV2Nl$a1Ul};_= z?PobfL^8X}%yo0bV>VW&u2)vJtu9TGzJ0Vs^!+N3k(S<$a8c%02z+;RZdn)^gotVo z1{PsuXUF#J(aM)~Je8T53HibT`LcEE))dMsv7wqkgPC?Jo;qch5&8iSAKyzjefpBwFgiTI=#4{#8O>d{c3I*NgCZ^R;=lN7vkDzJ4lIuE4zn=j{mtZ1sLs$wt;IpVe=%!g!U@8!+H#$KE(3~X|%7FAQ@$q{#L zN##by{X~*Agp7@i5eV4Hd~3^dNv*B=bB{e__HOSjw`YMB+|DC=`QE*~XAhd|!~FR8 z_+V}5F($YMJYDKqp64&E)$u*PG;18L#Q9MizAjh-arydnR*j20Z7LidKR(S;xIVZ6 z`9qfC6LR8|XW>^5JwXc_wE$g%13l{hqmSaUg!# zjYh}&`Q4OqD)UK$IdQm8ZeP>1$Ki0jow>QWBR~`Kg)d)rbbq~dL*S9w4vw^+7aF3! zkFHJ?GeCl|u~BJzRif_|-WkhKd)EjYCZWMRaFR-QW7#R#SmoLb`}iQQq6h2i?(Vw38r1=E|{@#%l6Kh&4Fnyf}Txn9EYy_yX2aTKPy=i-Trn`)RLyG(v zhK7c_{@|1P?E&_{z(9ao{Y#fFy`)T{W&z(%fKUONXTYj9kmY-06sQ9Jpy^lp^iU|^ z0LAml-n%J+>SBPj0Q#Kf$CkM?*S}f;8ZsRT``FlD?GE9vRAW_2tEzTwY-|9`ZT=c{ z%pwhLE@+L(sUPcZ!V|uOw&fy*U)$7l$KXAGX$WZ>8aE~-EzQZxdq{(M z_%cB3^@<9BqoM@rEVGZV@5szd3Zz&vWwaNvnZ+n4Xns%ys5%8Gx%A;fVU3HFHYaZJ z{rfiq)&V~_0LK%VcM2wh{7I&aSeqQ!!@PqZ0ZFKg)XGhCnc65)a zgoMOM$B_j+ZS9lOhYHStf~;_?cLO{ty^@tyjUDbExb4ZgYu5?F^Sl&FsU)adl#7cf z#BvbSMk7YzNlS}%W#3ri$<3u#t`Fur$E64#i*o5IMM3}LGPq5MqN9k1!n+e z_e)6pl}E#Gjx0Pl4U$XfHu0qn3Uy{0nToNtPQ0MxIZ8S5sgaJmq1U$Wj4SPRu^k}) z{qO}!``Lr_>jJN18T%_eU9Ydgu{vtm2Q*m;XfiP%BTeC$MeVN(??rt3D$caHaSF9d zKwei9U-EyQAL8LS_H|dz*|R7Eu`}+p98=Kmy95NbLynjfAvq76H451}^y<|KfM@rn zgl!DbaLv4!^7gGmSGA!8L(+0+{oGYjQsM=YrZt`K^kMsZ*T1!wr>3%rA3V4piR2X) z7UtE8J9^K5=)fO6>rovtWu~X6ciFsHbGo~XgZSe7NsuS3Q=PDH=dT%~(bfJ#swTyn zLf_v-%Q3U_o_PAy@G6y>LeZCM!$=GWX~hYC{`@&&@`C4wxw*7z_ck$QWv=aeWsdv! zNSPHOM_wNVP?x`8!YB|xWi#pyS;zY>5I+7kaR=~nUIjc z$j!yCUlSg?G$wQI-^I;+W_sF-Q5O}e1?m%t2ZBRFEFB!ub8>c{%grqT!P2;VWilVW zf=+8`X?gncB??-p2EI#=T$*%!c1|)mHMI#wk|&Moff5lF6-B@_C8wpGZuih3k*Yr| zuzK^Ma}yZk1x#G+*(odzxtenD(ut%b7Q31N&?2wyhx&q?YM!r!%8uyc zuo|8;&yf(ILG zbEkIiQGtmc`9lgxfZlG4A09InT-@kXem;+WIxneviCsif5Y`&X7x=9Gwef=l3dePQ zZCTdL{EZ;Is5+y(xN@b7j*~ViRNY}~Yg-$$st6kDG*HbIkl*o98z3MS7z00e@W29x zJM-d&38+x+9Pxbz4lu(jg>)U>x5kamb(pQOpGn*yP!RhN*rw)Plr&kxF-^9!t&JuA z$I$zr**7$zh`EW$_%~ynDr~z%z`fjeCT3=4Y1{7_!7$my#qr&pq-Y5n$oTG`J=U{6 zZ=Kq?@pSR}^z`(-{?yW-XbDd@T8>P>TVoc0ml$b($$(06wLB?xy%~FNG<2~9y^?ivgWfk=c}B{*6FA%m-FDSxA0+prtm z^D;;}bVoqtr!+v&f_txje5%j1ry<@t4R1%|n&awoi9}*5WxThx;XddaW|A~cGoW&? z1=Ev!tsI*p9zd09n#V;0eYcfkkJL8!fHpo;o9iP`V-pHWg|!Cry?GuNi>I@NL)jTX zeKL#_phZ?zR%7`&!5sMK_k~k$ zYjgQC^aH5k+XE^`tFeyh_L6mMH8u)(x-2x*8~rwGCxbgUYHe)XJm0g5rz--(;cu6p zu$PqNtg)GLr9B&OEk7rKplz$MiLAy7L8m(jLpP4%>0w@L?H%w}>d=jKJpJ(3q4IND zE$u=<(RYQ=%W^@|689-XzRzn-X$3EdP$akmBmy%x{#;#R6ua5`4Vql!Mdjsp9gF^^ z2Ws%u!`Nsj;l@7IFC@a&3#gx0Wmq4KyFdlthv-uTQ()163ZOn`f}&T)%xou+7wSD$ ztZ+b4q-How;ONn#LEneNo59@xtelE@d;BbQZT(KuMOI#>eqi_xITRU!GkQTtEHg~dIyyQo^1M~xbpWnHeGnJqeQ9*m?c>|JJs|260Rn4DW3Omd2LTs<>rFE= z3VJCkdMu_kPYQjvx*CX~ea*N11k+}D-(H!RpP$f>Muwe6qm^FCZVl3e(v24I?g1Y}sy)|R&dJZ83RVpkhfARl@^9Ewy1_-(ZQD+Y z8}S|qlhIuHuo-x!1u63E@(RXZgEz&jtoe4QW*Q(CUtGk&nc_tTi8AVxt5?N=7*E1@7dcvTXMu>j!0pg8FxbC; zzb;v@gd%(_6|^|{(_22M#<+x%xK1p6mZo+jxfw6d*Sv2nuX&dWf?;g7=LRtNvs`JL z#n))L5=I^NFs2Tu9x*kRDe147kni(vGlYX)|9_Yv!Au-sF>I$ZEbDY_>6sZ%u~&ArBk|wG!P{`p!p2V&BJGweO`jP?z&+k2NdN@ z=E~um(iJY9Y^GA9X5UDztgcGVY7-9QtKGK#TwnhTHB2f+XXl3)WYUePW&*V!E9(Rh zM`>+s|4iQm1soYtpJTY|SZfBT8X=@e$HcSMAR`)VU)~J;GA9;CHNrElKUz}tLrg72 zda`+jI#?6v9J4l4W>z1w)&K?oJFon4GP@5`N79^nv}Tbtop_s1NP~PlAfV!7zG?j7 zBS!?FE`X}0^vWw!$b{g@7$g<@lBo2cp~2GH`oP>oj=!u!EyCQ~{GqskE3DCno2yw3 zSRg-!d>op{9^15OlZCAu4ajB++Su5z$jQldDX7P_fSLuh;v5nU$7R4FWaZ%C5o$h6 z(mZEgv6@B2Izz=R<9Pqgi2QgAI#1>?6lHqg4i#}MXSr(%%J#(m_~yE@II{DMXkXOV zE`!fKHqI&FDN=`B<6Z?!;)(_jq=xYc{#Oz!Vrzt z4GIEprQpe`g;w-ba|OoK+}wanzVp&X|Jl;pkj|0Ck>*l&FSaS&BOfnt)HBLQQ3;8b zYfD**{sXtRa&mJ1_iiEj?jINCu8M%&y?==bKj{lmCfs zt#LEQhx&zwA>xJ4xp{c(7eq}#J}dYSDucg}>*=2i0xT(o|>K{kpItU_V5C|MX{qks>ED zsDof~;xw~jKaZk*{&@c@(X8zD#<`2?AqQsX<_d?s!K+|IfoKE|SRwL@2iPB1=yW!) zR%T{qs_-%qqbUa$flEq3k>!Ywd>puqII*@C^>a+*m&L@7TIc)ZZQxaxx?ks?I=#fy zkEa8^QO0+d-oMYEFXw!3qyE7IL7)j(5cqugNm*z>w3k-!Fbqu-7TqOUmNyjq|z zNO_NS?;>by79U8MmPx^Gr~-4OTrxk1vjtDydRRPY{K6EMscA0PjwuTSkqEzl$Y zJ-h;(7J$EfvxWra8@?e*1;MaFfKS_^7aEeFG>4UhLj#7-4pIT?)WiY5R!|E;FeCZz z%?wCZp@NLqYahGQ0ZkxMY3rNEq1FW}=HbSgFRU*>f=Ye2$pyu&fe%*?fMABIp?qp{ z+aN`v(R5N#qXKd5PhsHEh!sD_6FzTY0MP8JgL>j}^ITB>#Li~07R zRK(H2;^IzgZEdB03XL#@sy7jI$OyopzQ6yxNSq^NC1WmQolBr@-Rt-bTu>L)Lc;So zA8;yLmO8cMes8{wm0QE!ntoC}V5Ydfdk{&19UcXS%3`m5U<#$gr-)w@netuj+x;;pdLBjKx{+OdZa0uW*APf{{ za2?khGuHi;s2zGJ@RaVc@$($E#@vXe z=4OzFV~Z16OCG;{no~i4NUR*oblJWp;E<$bHiHAfpw!M3J_ao2Tf{Mu69OEGLo9yz zqT}dz(J$YCa)U}y z|MqaEnTt3GLA?x;KRLHhalLyc!p{h7GR0SSeQmlFraAspoDzB)_hX*XKuSsqJf(BDZYjPfQr{E)mc(LRZYQC!b6ex; zio0JunJ}i!|Hdq&O z5Dtc3m6?mcerHi4zGyTMiA`Xz7A4FIxVrzIzW~F60?g*w%*+l)KjN=nzt%I@zhg8e z`N|-sp$ps;#`%EfrAX$74<9n>2ylU_T^?_`dhOcilaxKOFRd&r5Mcal-M&{8tfR+| zA5XugZa-jh{siEysGM9UHIh<^>4V=Bg))8%@U(%X=>SF4nmwGI0JnmNqYDSaupM@Q zR%M(P^D1nDl8wPmU}YqkYp!m&aNz>O5h9Tkz+JZh1FxWfAJLb|<^*5CLIUNmAGilv zt&E>FXE?PF0KotMj!{=7h;wQ?rr`omD@a$t=x-mjF__(up9$XG*G6W_iuIu&TKhJ? zT}w-gJlDH}p}=8Fb8>MdfCX0yx(ZaV0LaQ$Am5=B9b=1.4-0, <1.7-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiGroup for PodSecurityPolicy. +*/}} +{{- define "podSecurityPolicy.apiGroup" -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "policy" -}} +{{- else -}} +{{- print "extensions" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodSecurityPolicy. +*/}} +{{- define "podSecurityPolicy.apiVersion" -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "extensions/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Redis(TM) image name +*/}} +{{- define "redis.image" -}} +{{- $registryName := .Values.image.registry -}} +{{- $repositoryName := .Values.image.repository -}} +{{- $tag := .Values.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Redis(TM) Sentinel image name +*/}} +{{- define "sentinel.image" -}} +{{- $registryName := .Values.sentinel.image.registry -}} +{{- $repositoryName := .Values.sentinel.image.repository -}} +{{- $tag := .Values.sentinel.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper image name (for the metrics image) +*/}} +{{- define "redis.metrics.image" -}} +{{- $registryName := .Values.metrics.image.registry -}} +{{- $repositoryName := .Values.metrics.image.repository -}} +{{- $tag := .Values.metrics.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper image name (for the sentinel metrics image) +*/}} +{{- define "sentinel.metrics.image" -}} +{{- $registryName := .Values.sentinel.metrics.image.registry -}} +{{- $repositoryName := .Values.sentinel.metrics.image.repository -}} +{{- $tag := .Values.sentinel.metrics.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "redis.volumePermissions.image" -}} +{{- $registryName := .Values.volumePermissions.image.registry -}} +{{- $repositoryName := .Values.volumePermissions.image.repository -}} +{{- $tag := .Values.volumePermissions.image.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the cert file. +*/}} +{{- define "redis.tlsCert" -}} +{{- required "Certificate filename is required when TLS in enabled" .Values.tls.certFilename | printf "/opt/bitnami/redis/certs/%s" -}} +{{- end -}} + +{{/* +Return the path to the cert key file. +*/}} +{{- define "redis.tlsCertKey" -}} +{{- required "Certificate Key filename is required when TLS in enabled" .Values.tls.certKeyFilename | printf "/opt/bitnami/redis/certs/%s" -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "redis.tlsCACert" -}} +{{- required "Certificate CA filename is required when TLS in enabled" .Values.tls.certCAFilename | printf "/opt/bitnami/redis/certs/%s" -}} +{{- end -}} + +{{/* +Return the path to the DH params file. +*/}} +{{- define "redis.tlsDHParams" -}} +{{- if .Values.tls.dhParamsFilename -}} +{{- printf "/opt/bitnami/redis/certs/%s" .Values.tls.dhParamsFilename -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "redis.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "redis.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Get the password secret. +*/}} +{{- define "redis.secretName" -}} +{{- if .Values.existingSecret -}} +{{- printf "%s" .Values.existingSecret -}} +{{- else -}} +{{- printf "%s" (include "redis.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the password key to be retrieved from Redis(TM) secret. +*/}} +{{- define "redis.secretPasswordKey" -}} +{{- if and .Values.existingSecret .Values.existingSecretPasswordKey -}} +{{- printf "%s" .Values.existingSecretPasswordKey -}} +{{- else -}} +{{- printf "redis-password" -}} +{{- end -}} +{{- end -}} + +{{/* +Return Redis(TM) password +*/}} +{{- define "redis.password" -}} +{{- if not (empty .Values.global.redis.password) }} + {{- .Values.global.redis.password -}} +{{- else if not (empty .Values.password) -}} + {{- .Values.password -}} +{{- else -}} + {{- randAlphaNum 10 -}} +{{- end -}} +{{- end -}} + +{{/* +Return sysctl image +*/}} +{{- define "redis.sysctl.image" -}} +{{- $registryName := default "docker.io" .Values.sysctlImage.registry -}} +{{- $repositoryName := .Values.sysctlImage.repository -}} +{{- $tag := default "buster" .Values.sysctlImage.tag | toString -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 doesn't support it, so we need to implement this if-else logic. +Also, we can't use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} + {{- if .Values.global.imageRegistry }} + {{- printf "%s/%s:%s" .Values.global.imageRegistry $repositoryName $tag -}} + {{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} + {{- end -}} +{{- else -}} + {{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "redis.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if or .Values.image.pullSecrets .Values.metrics.image.pullSecrets .Values.sysctlImage.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.metrics.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.sysctlImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- else if or .Values.image.pullSecrets .Values.metrics.image.pullSecrets .Values.sysctlImage.pullSecrets .Values.volumePermissions.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.metrics.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.sysctlImage.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- range .Values.volumePermissions.image.pullSecrets }} + - name: {{ . }} +{{- end }} +{{- end -}} +{{- end -}} + +{{/* Check if there are rolling tags in the images */}} +{{- define "redis.checkRollingTags" -}} +{{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .Values.image.repository }}:{{ .Values.image.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} +{{- if and (contains "bitnami/" .Values.sentinel.image.repository) (not (.Values.sentinel.image.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .Values.sentinel.image.repository }}:{{ .Values.sentinel.image.tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} +{{- end -}} + +{{/* +Return the proper Storage Class for master +*/}} +{{- define "redis.master.storageClass" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +*/}} +{{- if .Values.global -}} + {{- if .Values.global.storageClass -}} + {{- if (eq "-" .Values.global.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.global.storageClass -}} + {{- end -}} + {{- else -}} + {{- if .Values.master.persistence.storageClass -}} + {{- if (eq "-" .Values.master.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.master.persistence.storageClass -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.master.persistence.storageClass -}} + {{- if (eq "-" .Values.master.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.master.persistence.storageClass -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Storage Class for slave +*/}} +{{- define "redis.slave.storageClass" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +*/}} +{{- if .Values.global -}} + {{- if .Values.global.storageClass -}} + {{- if (eq "-" .Values.global.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.global.storageClass -}} + {{- end -}} + {{- else -}} + {{- if .Values.slave.persistence.storageClass -}} + {{- if (eq "-" .Values.slave.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.slave.persistence.storageClass -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.slave.persistence.storageClass -}} + {{- if (eq "-" .Values.slave.persistence.storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" .Values.slave.persistence.storageClass -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Compile all warnings into a single message, and call fail. +*/}} +{{- define "redis.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "redis.validateValues.spreadConstraints" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message | fail -}} +{{- end -}} +{{- end -}} + +{{/* Validate values of Redis(TM) - spreadConstrainsts K8s version */}} +{{- define "redis.validateValues.spreadConstraints" -}} +{{- if and (semverCompare "<1.16-0" .Capabilities.KubeVersion.GitVersion) .Values.slave.spreadConstraints -}} +redis: spreadConstraints + Pod Topology Spread Constraints are only available on K8s >= 1.16 + Find more information at https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +{{- end -}} +{{- end -}} + +{{/* +Renders a value that contains template. +Usage: +{{ include "redis.tplValue" (dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "redis.tplValue" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/configmap-scripts.yaml b/scripts/helm/helm/databases/charts/redis/templates/configmap-scripts.yaml new file mode 100644 index 000000000..6d7402721 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/configmap-scripts.yaml @@ -0,0 +1,430 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "redis.fullname" . }}-scripts + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: +{{- if and .Values.cluster.enabled .Values.sentinel.enabled }} + start-node.sh: | + #!/bin/bash + + . /opt/bitnami/scripts/libos.sh + . /opt/bitnami/scripts/liblog.sh + . /opt/bitnami/scripts/libvalidations.sh + + not_exists_dns_entry() { + myip=$(hostname -i) + + if [[ -z "$(getent ahosts "$HEADLESS_SERVICE" | grep "^${myip}" )" ]]; then + warn "$HEADLESS_SERVICE does not contain the IP of this pod: ${myip}" + return 1 + fi + info "$HEADLESS_SERVICE has my IP: ${myip}" + return 0 + } + + HEADLESS_SERVICE="{{ template "redis.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + REDIS_SERVICE="{{ template "redis.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + + # Waits for DNS to add this ip to the service DNS entry + retry_while not_exists_dns_entry + + export REDIS_REPLICATION_MODE="slave" + if [[ -z "$(getent ahosts "$HEADLESS_SERVICE" | grep -v "^$(hostname -i) ")" ]]; then + export REDIS_REPLICATION_MODE="master" + fi + + {{- if and .Values.securityContext.runAsUser (eq (.Values.securityContext.runAsUser | int) 0) }} + useradd redis + chown -R redis {{ .Values.slave.persistence.path }} + {{- end }} + + if [[ -n $REDIS_PASSWORD_FILE ]]; then + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux + fi + + if [[ -n $REDIS_MASTER_PASSWORD_FILE ]]; then + password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}` + export REDIS_MASTER_PASSWORD=$password_aux + fi + + if [[ "$REDIS_REPLICATION_MODE" == "master" ]]; then + echo "I am master" + if [[ ! -f /opt/bitnami/redis/etc/master.conf ]];then + cp /opt/bitnami/redis/mounted-etc/master.conf /opt/bitnami/redis/etc/master.conf + fi + else + if [[ ! -f /opt/bitnami/redis/etc/replica.conf ]];then + cp /opt/bitnami/redis/mounted-etc/replica.conf /opt/bitnami/redis/etc/replica.conf + fi + + if is_boolean_yes "$REDIS_TLS_ENABLED"; then + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_SERVICE -p {{ .Values.sentinel.port }} --tls --cert ${REDIS_TLS_CERT_FILE} --key ${REDIS_TLS_KEY_FILE} --cacert ${REDIS_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + else + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_SERVICE -p {{ .Values.sentinel.port }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + fi + REDIS_SENTINEL_INFO=($($sentinel_info_command)) + REDIS_MASTER_HOST=${REDIS_SENTINEL_INFO[0]} + REDIS_MASTER_PORT_NUMBER=${REDIS_SENTINEL_INFO[1]} + + + # Immediately attempt to connect to the reported master. If it doesn't exist the connection attempt will either hang + # or fail with "port unreachable" and give no data. The liveness check will then timeout waiting for the redis + # container to be ready and restart the it. By then the new master will likely have been elected + if is_boolean_yes "$REDIS_TLS_ENABLED"; then + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_MASTER_HOST -p {{ .Values.sentinel.port }} --tls --cert ${REDIS_TLS_CERT_FILE} --key ${REDIS_TLS_KEY_FILE} --cacert ${REDIS_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + else + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_MASTER_HOST -p {{ .Values.sentinel.port }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + fi + + if [[ ! ($($sentinel_info_command)) ]]; then + # master doesn't actually exist, this probably means the remaining pods haven't elected a new one yet + # and are reporting the old one still. Once this happens the container will get stuck and never see the new + # master. We stop here to allow the container to not pass the liveness check and be restarted. + exit 1 + fi + fi + + if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then + cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf + fi + {{- if .Values.tls.enabled }} + ARGS=("--port" "0") + ARGS+=("--tls-port" "${REDIS_TLS_PORT}") + ARGS+=("--tls-cert-file" "${REDIS_TLS_CERT_FILE}") + ARGS+=("--tls-key-file" "${REDIS_TLS_KEY_FILE}") + ARGS+=("--tls-ca-cert-file" "${REDIS_TLS_CA_FILE}") + ARGS+=("--tls-auth-clients" "${REDIS_TLS_AUTH_CLIENTS}") + ARGS+=("--tls-replication" "yes") + {{- if .Values.tls.dhParamsFilename }} + ARGS+=("--tls-dh-params-file" "${REDIS_TLS_DH_PARAMS_FILE}") + {{- end }} + {{- else }} + ARGS=("--port" "${REDIS_PORT}") + {{- end }} + + if [[ "$REDIS_REPLICATION_MODE" == "slave" ]]; then + ARGS+=("--slaveof" "${REDIS_MASTER_HOST}" "${REDIS_MASTER_PORT_NUMBER}") + fi + + {{- if .Values.usePassword }} + ARGS+=("--requirepass" "${REDIS_PASSWORD}") + ARGS+=("--masterauth" "${REDIS_MASTER_PASSWORD}") + {{- else }} + ARGS+=("--protected-mode" "no") + {{- end }} + + if [[ "$REDIS_REPLICATION_MODE" == "master" ]]; then + ARGS+=("--include" "/opt/bitnami/redis/etc/master.conf") + else + ARGS+=("--include" "/opt/bitnami/redis/etc/replica.conf") + fi + + ARGS+=("--include" "/opt/bitnami/redis/etc/redis.conf") + {{- if .Values.slave.extraFlags }} + {{- range .Values.slave.extraFlags }} + ARGS+=({{ . | quote }}) + {{- end }} + {{- end }} + + {{- if .Values.slave.preExecCmds }} + {{ .Values.slave.preExecCmds | nindent 4}} + {{- end }} + + {{- if .Values.slave.command }} + exec {{ .Values.slave.command }} "${ARGS[@]}" + {{- else }} + exec redis-server "${ARGS[@]}" + {{- end }} + + start-sentinel.sh: | + #!/bin/bash + + . /opt/bitnami/scripts/libos.sh + . /opt/bitnami/scripts/libvalidations.sh + . /opt/bitnami/scripts/libfile.sh + + sentinel_conf_set() { + local -r key="${1:?missing key}" + local value="${2:-}" + + # Sanitize inputs + value="${value//\\/\\\\}" + value="${value//&/\\&}" + value="${value//\?/\\?}" + [[ "$value" = "" ]] && value="\"$value\"" + + replace_in_file "/opt/bitnami/redis-sentinel/etc/sentinel.conf" "^#*\s*${key} .*" "${key} ${value}" false + } + sentinel_conf_add() { + echo $'\n'"$@" >> "/opt/bitnami/redis-sentinel/etc/sentinel.conf" + } + host_id() { + echo "$1" | openssl sha1 | awk '{print $2}' + } + not_exists_dns_entry() { + myip=$(hostname -i) + + if [[ -z "$(getent ahosts "$HEADLESS_SERVICE" | grep "^${myip}" )" ]]; then + warn "$HEADLESS_SERVICE does not contain the IP of this pod: ${myip}" + return 1 + fi + info "$HEADLESS_SERVICE has my IP: ${myip}" + return 0 + } + + HEADLESS_SERVICE="{{ template "redis.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + REDIS_SERVICE="{{ template "redis.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + + if [[ -n $REDIS_PASSWORD_FILE ]]; then + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux + fi + + if [[ ! -f /opt/bitnami/redis-sentinel/etc/sentinel.conf ]]; then + cp /opt/bitnami/redis-sentinel/mounted-etc/sentinel.conf /opt/bitnami/redis-sentinel/etc/sentinel.conf + {{- if .Values.usePassword }} + printf "\nsentinel auth-pass %s %s" "{{ .Values.sentinel.masterSet }}" "$REDIS_PASSWORD" >> /opt/bitnami/redis-sentinel/etc/sentinel.conf + {{- if .Values.sentinel.usePassword }} + printf "\nrequirepass %s" "$REDIS_PASSWORD" >> /opt/bitnami/redis-sentinel/etc/sentinel.conf + {{- end }} + {{- end }} + {{- if .Values.sentinel.staticID }} + printf "\nsentinel myid %s" "$(host_id "$HOSTNAME")" >> /opt/bitnami/redis-sentinel/etc/sentinel.conf + {{- end }} + fi + + export REDIS_REPLICATION_MODE="slave" + + # Waits for DNS to add this ip to the service DNS entry + retry_while not_exists_dns_entry + + if [[ -z "$(getent ahosts "$HEADLESS_SERVICE" | grep -v "^$(hostname -i)")" ]]; then + export REDIS_REPLICATION_MODE="master" + fi + + # Clean sentineles from the current sentinel nodes + for node in $( getent ahosts "$HEADLESS_SERVICE" | grep -v "^$(hostname -i)" | cut -f 1 -d ' ' | uniq ); do + info "Cleaning sentinels in sentinel node: $node" + if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then + redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $node -p {{ .Values.sentinel.port }} --tls --cert ${REDIS_SENTINEL_TLS_CERT_FILE} --key ${REDIS_SENTINEL_TLS_KEY_FILE} --cacert ${REDIS_SENTINEL_TLS_CA_FILE} sentinel reset "*" + else + redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $node -p {{ .Values.sentinel.port }} sentinel reset "*" + fi + sleep {{ .Values.sentinel.cleanDelaySeconds }} + done + info "Sentinels clean up done" + + if [[ "$REDIS_REPLICATION_MODE" == "master" ]]; then + REDIS_MASTER_HOST="$(hostname -i)" + REDIS_MASTER_PORT_NUMBER="{{ .Values.redisPort }}" + else + if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_SERVICE -p {{ .Values.sentinel.port }} --tls --cert ${REDIS_SENTINEL_TLS_CERT_FILE} --key ${REDIS_SENTINEL_TLS_KEY_FILE} --cacert ${REDIS_SENTINEL_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + else + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_SERVICE -p {{ .Values.sentinel.port }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + fi + REDIS_SENTINEL_INFO=($($sentinel_info_command)) + REDIS_MASTER_HOST=${REDIS_SENTINEL_INFO[0]} + REDIS_MASTER_PORT_NUMBER=${REDIS_SENTINEL_INFO[1]} + + # Immediately attempt to connect to the reported master. If it doesn't exist the connection attempt will either hang + # or fail with "port unreachable" and give no data. The liveness check will then timeout waiting for the sentinel + # container to be ready and restart the it. By then the new master will likely have been elected + if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_MASTER_HOST -p {{ .Values.sentinel.port }} --tls --cert ${REDIS_SENTINEL_TLS_CERT_FILE} --key ${REDIS_SENTINEL_TLS_KEY_FILE} --cacert ${REDIS_SENTINEL_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + else + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a $REDIS_PASSWORD {{- end }} -h $REDIS_MASTER_HOST -p {{ .Values.sentinel.port }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + fi + + if [[ ! ($($sentinel_info_command)) ]]; then + # master doesn't actually exist, this probably means the remaining pods haven't elected a new one yet + # and are reporting the old one still. Once this happens the container will get stuck and never see the new + # master. We stop here to allow the container to not pass the liveness check and be restarted. + exit 1 + fi + fi + sentinel_conf_set "sentinel monitor" "{{ .Values.sentinel.masterSet }} "$REDIS_MASTER_HOST" "$REDIS_MASTER_PORT_NUMBER" {{ .Values.sentinel.quorum }}" + + add_replica() { + if [[ "$1" != "$REDIS_MASTER_HOST" ]]; then + sentinel_conf_add "sentinel known-replica {{ .Values.sentinel.masterSet }} $1 {{ .Values.redisPort }}" + fi + } + + {{- if .Values.sentinel.staticID }} + # remove generated known sentinels and replicas + tmp="$(sed -e '/^sentinel known-/d' -e '/^$/d' /opt/bitnami/redis-sentinel/etc/sentinel.conf)" + echo "$tmp" > /opt/bitnami/redis-sentinel/etc/sentinel.conf + + for node in $(seq 0 {{ .Values.cluster.slaveCount }}); do + NAME="{{ template "redis.fullname" . }}-node-$node" + IP="$(getent hosts "$NAME.$HEADLESS_SERVICE" | awk ' {print $1 }')" + if [[ "$NAME" != "$HOSTNAME" && -n "$IP" ]]; then + sentinel_conf_add "sentinel known-sentinel {{ .Values.sentinel.masterSet }} $IP {{ .Values.sentinel.port }} $(host_id "$NAME")" + add_replica "$IP" + fi + done + add_replica "$(hostname -i)" + {{- end }} + + {{- if .Values.tls.enabled }} + ARGS=("--port" "0") + ARGS+=("--tls-port" "${REDIS_SENTINEL_TLS_PORT_NUMBER}") + ARGS+=("--tls-cert-file" "${REDIS_SENTINEL_TLS_CERT_FILE}") + ARGS+=("--tls-key-file" "${REDIS_SENTINEL_TLS_KEY_FILE}") + ARGS+=("--tls-ca-cert-file" "${REDIS_SENTINEL_TLS_CA_FILE}") + ARGS+=("--tls-replication" "yes") + ARGS+=("--tls-auth-clients" "${REDIS_SENTINEL_TLS_AUTH_CLIENTS}") + {{- if .Values.tls.dhParamsFilename }} + ARGS+=("--tls-dh-params-file" "${REDIS_SENTINEL_TLS_DH_PARAMS_FILE}") + {{- end }} + {{- end }} + {{- if .Values.sentinel.preExecCmds }} + {{ .Values.sentinel.preExecCmds | nindent 4 }} + {{- end }} + exec redis-server /opt/bitnami/redis-sentinel/etc/sentinel.conf --sentinel {{- if .Values.tls.enabled }} "${ARGS[@]}" {{- end }} + prestop-sentinel.sh: | + #!/bin/bash + + . /opt/bitnami/scripts/libvalidations.sh + + REDIS_SERVICE="{{ include "redis.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" + + if [[ -n "$REDIS_PASSWORD_FILE" ]]; then + password_aux=$(cat "$REDIS_PASSWORD_FILE") + export REDIS_PASSWORD="$password_aux" + fi + + if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a ${REDIS_PASSWORD} {{- end }} -h ${REDIS_SERVICE} -p {{ .Values.sentinel.port }} --tls --cert ${REDIS_SENTINEL_TLS_CERT_FILE} --key ${REDIS_SENTINEL_TLS_KEY_FILE} --cacert ${REDIS_SENTINEL_TLS_CA_FILE} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + else + sentinel_info_command="redis-cli {{- if .Values.usePassword }} -a ${REDIS_PASSWORD} {{- end }} -h ${REDIS_SERVICE} -p {{ .Values.sentinel.port }} sentinel get-master-addr-by-name {{ .Values.sentinel.masterSet }}" + fi + REDIS_SENTINEL_INFO=($($sentinel_info_command)) + REDIS_MASTER_HOST="${REDIS_SENTINEL_INFO[0]}" + + if [[ "$REDIS_MASTER_HOST" == "$(hostname -i)" ]]; then + if is_boolean_yes "$REDIS_SENTINEL_TLS_ENABLED"; then + redis-cli {{- if .Values.usePassword }} -a "$REDIS_PASSWORD" {{- end }} -h "$REDIS_SERVICE" -p {{ .Values.sentinel.port }} --tls --cert "$REDIS_SENTINEL_TLS_CERT_FILE" --key "$REDIS_SENTINEL_TLS_KEY_FILE" --cacert "$REDIS_SENTINEL_TLS_CA_FILE" sentinel failover mymaster + else + redis-cli {{- if .Values.usePassword }} -a "$REDIS_PASSWORD" {{- end }} -h "$REDIS_SERVICE" -p {{ .Values.sentinel.port }} sentinel failover mymaster + fi + fi +{{- else }} + start-master.sh: | + #!/bin/bash + {{- if and .Values.securityContext.runAsUser (eq (.Values.securityContext.runAsUser | int) 0) }} + useradd redis + chown -R redis {{ .Values.master.persistence.path }} + {{- end }} + if [[ -n $REDIS_PASSWORD_FILE ]]; then + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux + fi + if [[ ! -f /opt/bitnami/redis/etc/master.conf ]];then + cp /opt/bitnami/redis/mounted-etc/master.conf /opt/bitnami/redis/etc/master.conf + fi + if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then + cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf + fi + {{- if .Values.tls.enabled }} + ARGS=("--port" "0") + ARGS+=("--tls-port" "${REDIS_TLS_PORT}") + ARGS+=("--tls-cert-file" "${REDIS_TLS_CERT_FILE}") + ARGS+=("--tls-key-file" "${REDIS_TLS_KEY_FILE}") + ARGS+=("--tls-ca-cert-file" "${REDIS_TLS_CA_FILE}") + ARGS+=("--tls-auth-clients" "${REDIS_TLS_AUTH_CLIENTS}") + {{- if .Values.tls.dhParamsFilename }} + ARGS+=("--tls-dh-params-file" "${REDIS_TLS_DH_PARAMS_FILE}") + {{- end }} + {{- else }} + ARGS=("--port" "${REDIS_PORT}") + {{- end }} + {{- if .Values.usePassword }} + ARGS+=("--requirepass" "${REDIS_PASSWORD}") + ARGS+=("--masterauth" "${REDIS_PASSWORD}") + {{- else }} + ARGS+=("--protected-mode" "no") + {{- end }} + ARGS+=("--include" "/opt/bitnami/redis/etc/redis.conf") + ARGS+=("--include" "/opt/bitnami/redis/etc/master.conf") + {{- if .Values.master.extraFlags }} + {{- range .Values.master.extraFlags }} + ARGS+=({{ . | quote }}) + {{- end }} + {{- end }} + {{- if .Values.master.preExecCmds }} + {{ .Values.master.preExecCmds | nindent 4}} + {{- end }} + {{- if .Values.master.command }} + exec {{ .Values.master.command }} "${ARGS[@]}" + {{- else }} + exec redis-server "${ARGS[@]}" + {{- end }} + {{- if .Values.cluster.enabled }} + start-slave.sh: | + #!/bin/bash + {{- if and .Values.securityContext.runAsUser (eq (.Values.securityContext.runAsUser | int) 0) }} + useradd redis + chown -R redis {{ .Values.slave.persistence.path }} + {{- end }} + if [[ -n $REDIS_PASSWORD_FILE ]]; then + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux + fi + if [[ -n $REDIS_MASTER_PASSWORD_FILE ]]; then + password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}` + export REDIS_MASTER_PASSWORD=$password_aux + fi + if [[ ! -f /opt/bitnami/redis/etc/replica.conf ]];then + cp /opt/bitnami/redis/mounted-etc/replica.conf /opt/bitnami/redis/etc/replica.conf + fi + if [[ ! -f /opt/bitnami/redis/etc/redis.conf ]];then + cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf + fi + {{- if .Values.tls.enabled }} + ARGS=("--port" "0") + ARGS+=("--tls-port" "${REDIS_TLS_PORT}") + ARGS+=("--tls-cert-file" "${REDIS_TLS_CERT_FILE}") + ARGS+=("--tls-key-file" "${REDIS_TLS_KEY_FILE}") + ARGS+=("--tls-ca-cert-file" "${REDIS_TLS_CA_FILE}") + ARGS+=("--tls-auth-clients" "${REDIS_TLS_AUTH_CLIENTS}") + ARGS+=("--tls-replication" "yes") + {{- if .Values.tls.dhParamsFilename }} + ARGS+=("--tls-dh-params-file" "${REDIS_TLS_DH_PARAMS_FILE}") + {{- end }} + {{- else }} + ARGS=("--port" "${REDIS_PORT}") + {{- end }} + ARGS+=("--slaveof" "${REDIS_MASTER_HOST}" "${REDIS_MASTER_PORT_NUMBER}") + {{- if .Values.usePassword }} + ARGS+=("--requirepass" "${REDIS_PASSWORD}") + ARGS+=("--masterauth" "${REDIS_MASTER_PASSWORD}") + {{- else }} + ARGS+=("--protected-mode" "no") + {{- end }} + ARGS+=("--include" "/opt/bitnami/redis/etc/redis.conf") + ARGS+=("--include" "/opt/bitnami/redis/etc/replica.conf") + {{- if .Values.slave.extraFlags }} + {{- range .Values.slave.extraFlags }} + ARGS+=({{ . | quote }}) + {{- end }} + {{- end }} + {{- if .Values.slave.preExecCmds }} + {{ .Values.slave.preExecCmds | nindent 4}} + {{- end }} + {{- if .Values.slave.command }} + exec {{ .Values.slave.command }} "${ARGS[@]}" + {{- else }} + exec redis-server "${ARGS[@]}" + {{- end }} + {{- end }} + +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/configmap.yaml b/scripts/helm/helm/databases/charts/redis/templates/configmap.yaml new file mode 100644 index 000000000..77bdc81e8 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/configmap.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + redis.conf: |- +{{- if .Values.configmap }} + # User-supplied configuration: +{{- tpl .Values.configmap . | nindent 4 }} +{{- end }} + master.conf: |- + dir {{ .Values.master.persistence.path }} +{{- if .Values.master.configmap }} + # User-supplied master configuration: +{{- tpl .Values.master.configmap . | nindent 4 }} +{{- end }} +{{- if .Values.master.disableCommands }} +{{- range .Values.master.disableCommands }} + rename-command {{ . }} "" +{{- end }} +{{- end }} + replica.conf: |- + dir {{ .Values.slave.persistence.path }} + slave-read-only yes +{{- if .Values.slave.configmap }} + # User-supplied slave configuration: +{{- tpl .Values.slave.configmap . | nindent 4 }} +{{- end }} +{{- if .Values.slave.disableCommands }} +{{- range .Values.slave.disableCommands }} + rename-command {{ . }} "" +{{- end }} +{{- end }} +{{- if .Values.sentinel.enabled }} + sentinel.conf: |- + dir "/tmp" + bind 0.0.0.0 + port {{ .Values.sentinel.port }} + sentinel monitor {{ .Values.sentinel.masterSet }} {{ template "redis.fullname" . }}-node-0.{{ template "redis.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} {{ .Values.redisPort }} {{ .Values.sentinel.quorum }} + sentinel down-after-milliseconds {{ .Values.sentinel.masterSet }} {{ .Values.sentinel.downAfterMilliseconds }} + sentinel failover-timeout {{ .Values.sentinel.masterSet }} {{ .Values.sentinel.failoverTimeout }} + sentinel parallel-syncs {{ .Values.sentinel.masterSet }} {{ .Values.sentinel.parallelSyncs }} +{{- if .Values.sentinel.configmap }} + # User-supplied sentinel configuration: +{{- tpl .Values.sentinel.configmap . | nindent 4 }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/headless-svc.yaml b/scripts/helm/helm/databases/charts/redis/templates/headless-svc.yaml new file mode 100644 index 000000000..d758c0d23 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/headless-svc.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "redis.fullname" . }}-headless + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: ClusterIP + clusterIP: None + {{- if .Values.sentinel.enabled }} + publishNotReadyAddresses: true + {{- end }} + ports: + - name: tcp-redis + port: {{ .Values.redisPort }} + targetPort: redis + {{- if .Values.sentinel.enabled }} + - name: tcp-sentinel + port: {{ .Values.sentinel.port }} + targetPort: redis-sentinel + {{- end }} + selector: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/health-configmap.yaml b/scripts/helm/helm/databases/charts/redis/templates/health-configmap.yaml new file mode 100644 index 000000000..1bb8e74d9 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/health-configmap.yaml @@ -0,0 +1,176 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "redis.fullname" . }}-health + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + ping_readiness_local.sh: |- + #!/bin/bash +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux +{{- end }} + export REDISCLI_AUTH="$REDIS_PASSWORD" + response=$( + timeout -s 3 $1 \ + redis-cli \ + -h localhost \ +{{- if .Values.tls.enabled }} + -p $REDIS_TLS_PORT \ + --tls \ + --cacert {{ template "redis.tlsCACert" . }} \ + {{- if .Values.tls.authClients }} + --cert {{ template "redis.tlsCert" . }} \ + --key {{ template "redis.tlsCertKey" . }} \ + {{- end }} +{{- else }} + -p $REDIS_PORT \ +{{- end }} + ping + ) + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + ping_liveness_local.sh: |- + #!/bin/bash +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux +{{- end }} + export REDISCLI_AUTH="$REDIS_PASSWORD" + response=$( + timeout -s 3 $1 \ + redis-cli \ + -h localhost \ +{{- if .Values.tls.enabled }} + -p $REDIS_TLS_PORT \ + --tls \ + --cacert {{ template "redis.tlsCACert" . }} \ + {{- if .Values.tls.authClients }} + --cert {{ template "redis.tlsCert" . }} \ + --key {{ template "redis.tlsCertKey" . }} \ + {{- end }} +{{- else }} + -p $REDIS_PORT \ +{{- end }} + ping + ) + if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then + echo "$response" + exit 1 + fi +{{- if .Values.sentinel.enabled }} + ping_sentinel.sh: |- + #!/bin/bash +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_PASSWORD_FILE}` + export REDIS_PASSWORD=$password_aux +{{- end }} + export REDISCLI_AUTH="$REDIS_PASSWORD" + response=$( + timeout -s 3 $1 \ + redis-cli \ + -h localhost \ +{{- if .Values.tls.enabled }} + -p $REDIS_SENTINEL_TLS_PORT_NUMBER \ + --tls \ + --cacert {{ template "redis.tlsCACert" . }} \ + {{- if .Values.tls.authClients }} + --cert {{ template "redis.tlsCert" . }} \ + --key {{ template "redis.tlsCertKey" . }} \ + {{- end }} +{{- else }} + -p $REDIS_SENTINEL_PORT \ +{{- end }} + ping + ) + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + parse_sentinels.awk: |- + /ip/ {FOUND_IP=1} + /port/ {FOUND_PORT=1} + /runid/ {FOUND_RUNID=1} + !/ip|port|runid/ { + if (FOUND_IP==1) { + IP=$1; FOUND_IP=0; + } + else if (FOUND_PORT==1) { + PORT=$1; + FOUND_PORT=0; + } else if (FOUND_RUNID==1) { + printf "\nsentinel known-sentinel {{ .Values.sentinel.masterSet }} %s %s %s", IP, PORT, $0; FOUND_RUNID=0; + } + } +{{- end }} + ping_readiness_master.sh: |- + #!/bin/bash +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}` + export REDIS_MASTER_PASSWORD=$password_aux +{{- end }} + export REDISCLI_AUTH="$REDIS_MASTER_PASSWORD" + response=$( + timeout -s 3 $1 \ + redis-cli \ + -h $REDIS_MASTER_HOST \ + -p $REDIS_MASTER_PORT_NUMBER \ +{{- if .Values.tls.enabled }} + --tls \ + --cacert {{ template "redis.tlsCACert" . }} \ + {{- if .Values.tls.authClients }} + --cert {{ template "redis.tlsCert" . }} \ + --key {{ template "redis.tlsCertKey" . }} \ + {{- end }} +{{- end }} + ping + ) + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + ping_liveness_master.sh: |- + #!/bin/bash +{{- if .Values.usePasswordFile }} + password_aux=`cat ${REDIS_MASTER_PASSWORD_FILE}` + export REDIS_MASTER_PASSWORD=$password_aux +{{- end }} + export REDISCLI_AUTH="$REDIS_MASTER_PASSWORD" + response=$( + timeout -s 3 $1 \ + redis-cli \ + -h $REDIS_MASTER_HOST \ + -p $REDIS_MASTER_PORT_NUMBER \ +{{- if .Values.tls.enabled }} + --tls \ + --cacert {{ template "redis.tlsCACert" . }} \ + {{- if .Values.tls.authClients }} + --cert {{ template "redis.tlsCert" . }} \ + --key {{ template "redis.tlsCertKey" . }} \ + {{- end }} +{{- end }} + ping + ) + if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then + echo "$response" + exit 1 + fi + ping_readiness_local_and_master.sh: |- + script_dir="$(dirname "$0")" + exit_status=0 + "$script_dir/ping_readiness_local.sh" $1 || exit_status=$? + "$script_dir/ping_readiness_master.sh" $1 || exit_status=$? + exit $exit_status + ping_liveness_local_and_master.sh: |- + script_dir="$(dirname "$0")" + exit_status=0 + "$script_dir/ping_liveness_local.sh" $1 || exit_status=$? + "$script_dir/ping_liveness_master.sh" $1 || exit_status=$? + exit $exit_status diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-prometheus.yaml b/scripts/helm/helm/databases/charts/redis/templates/metrics-prometheus.yaml new file mode 100644 index 000000000..ed53dc6e2 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/metrics-prometheus.yaml @@ -0,0 +1,39 @@ +{{- if and (.Values.metrics.enabled) (.Values.metrics.serviceMonitor.enabled) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "redis.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- else }} + namespace: {{ .Release.Namespace | quote }} + {{- end }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- range $key, $value := .Values.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + endpoints: + - port: metrics + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelings }} + relabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.relabelings "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.metricRelabelings "context" $) | nindent 6 }} + {{- end }} + selector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + app.kubernetes.io/component: "metrics" + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml b/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml new file mode 100644 index 000000000..43cc2aa3c --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.sentinel.enabled .Values.sentinel.metrics.enabled .Values.sentinel.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "redis.fullname" . }}-sentinel-metrics + {{- if .Values.sentinel.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.sentinel.metrics.serviceMonitor.namespace }} + {{- else }} + namespace: {{ .Release.Namespace | quote }} + {{- end }} + labels: + app: {{ include "redis.name" . }} + chart: {{ include "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- range $key, $value := .Values.sentinel.metrics.serviceMonitor.selector }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + endpoints: + - port: sentinelmetrics + {{- if .Values.sentinel.metrics.serviceMonitor.interval }} + interval: {{ .Values.sentinel.metrics.serviceMonitor.interval }} + {{- end }} + selector: + matchLabels: + app: {{ include "redis.name" . }} + release: {{ .Release.Name }} + app.kubernetes.io/component: "sentinel-metrics" + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-svc.yaml b/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-svc.yaml new file mode 100644 index 000000000..25f3770de --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-svc.yaml @@ -0,0 +1,35 @@ +{{- if and .Values.sentinel.enabled .Values.sentinel.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "redis.fullname" . }}-sentinel-metrics + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ include "redis.name" . }} + chart: {{ include "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + app.kubernetes.io/component: "sentinel-metrics" + {{- if .Values.sentinel.metrics.service.labels -}} + {{- include "common.tplvalues.render" (dict "value" .Values.sentinel.metrics.service.labels "context" $) | nindent 4 }} + {{- end -}} + {{- if .Values.sentinel.metrics.service.annotations }} + annotations: {{- toYaml .Values.sentinel.metrics.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.sentinel.metrics.service.type }} + {{- if eq .Values.sentinel.metrics.service.type "LoadBalancer" }} + externalTrafficPolicy: {{ .Values.sentinel.metrics.service.externalTrafficPolicy }} + {{- end }} + {{- if and (eq .Values.sentinel.metrics.service.type "LoadBalancer") .Values.sentinel.metrics.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.sentinel.metrics.service.loadBalancerIP }} + {{- end }} + ports: + - port: {{ .Values.sentinel.metrics.service.port }} + targetPort: sentinelmetrics + protocol: TCP + name: sentinelmetrics + selector: + app: {{ include "redis.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-svc.yaml b/scripts/helm/helm/databases/charts/redis/templates/metrics-svc.yaml new file mode 100644 index 000000000..767a464e5 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/metrics-svc.yaml @@ -0,0 +1,34 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "redis.fullname" . }}-metrics + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + app.kubernetes.io/component: "metrics" + {{- if .Values.metrics.service.labels -}} + {{- toYaml .Values.metrics.service.labels | nindent 4 }} + {{- end -}} + {{- if .Values.metrics.service.annotations }} + annotations: {{- toYaml .Values.metrics.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.metrics.service.type }} + {{ if eq .Values.metrics.service.type "LoadBalancer" }} + externalTrafficPolicy: {{ .Values.metrics.service.externalTrafficPolicy }} + {{- end }} + {{ if and (eq .Values.metrics.service.type "LoadBalancer") .Values.metrics.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.metrics.service.loadBalancerIP }} + {{- end }} + ports: + - name: metrics + port: 9121 + targetPort: metrics + selector: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/networkpolicy.yaml b/scripts/helm/helm/databases/charts/redis/templates/networkpolicy.yaml new file mode 100644 index 000000000..0249bc0e6 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/networkpolicy.yaml @@ -0,0 +1,74 @@ +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: {{ template "networkPolicy.apiVersion" . }} +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + podSelector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + {{- if .Values.cluster.enabled }} + policyTypes: + - Ingress + - Egress + egress: + # Allow dns resolution + - ports: + - port: 53 + protocol: UDP + # Allow outbound connections to other cluster pods + - ports: + - port: {{ .Values.redisPort }} + {{- if .Values.sentinel.enabled }} + - port: {{ .Values.sentinel.port }} + {{- end }} + to: + - podSelector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + {{- end }} + ingress: + # Allow inbound connections + - ports: + - port: {{ .Values.redisPort }} + {{- if .Values.sentinel.enabled }} + - port: {{ .Values.sentinel.port }} + {{- end }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "redis.fullname" . }}-client: "true" + - podSelector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + {{- if .Values.networkPolicy.ingressNSMatchLabels }} + - namespaceSelector: + matchLabels: + {{- range $key, $value := .Values.networkPolicy.ingressNSMatchLabels }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- if .Values.networkPolicy.ingressNSPodMatchLabels }} + podSelector: + matchLabels: + {{- range $key, $value := .Values.networkPolicy.ingressNSPodMatchLabels }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.metrics.enabled }} + # Allow prometheus scrapes for metrics + - ports: + - port: 9121 + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/pdb.yaml b/scripts/helm/helm/databases/charts/redis/templates/pdb.yaml new file mode 100644 index 000000000..b9dc54b36 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/pdb.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/prometheusrule.yaml b/scripts/helm/helm/databases/charts/redis/templates/prometheusrule.yaml new file mode 100644 index 000000000..48ae017f6 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/prometheusrule.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ template "redis.fullname" . }} + {{- if .Values.metrics.prometheusRule.namespace }} + namespace: {{ .Values.metrics.prometheusRule.namespace }} + {{- else }} + namespace: {{ .Release.Namespace | quote }} + {{- end }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +{{- with .Values.metrics.prometheusRule.additionalLabels }} +{{- toYaml . | nindent 4 }} +{{- end }} +spec: +{{- with .Values.metrics.prometheusRule.rules }} + groups: + - name: {{ template "redis.name" $ }} + rules: {{- tpl (toYaml .) $ | nindent 8 }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/psp.yaml b/scripts/helm/helm/databases/charts/redis/templates/psp.yaml new file mode 100644 index 000000000..eca04c134 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/psp.yaml @@ -0,0 +1,43 @@ +{{- if .Values.podSecurityPolicy.create }} +apiVersion: {{ template "podSecurityPolicy.apiVersion" . }} +kind: PodSecurityPolicy +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + allowPrivilegeEscalation: false + fsGroup: + rule: 'MustRunAs' + ranges: + - min: {{ .Values.securityContext.fsGroup }} + max: {{ .Values.securityContext.fsGroup }} + hostIPC: false + hostNetwork: false + hostPID: false + privileged: false + readOnlyRootFilesystem: false + requiredDropCapabilities: + - ALL + runAsUser: + rule: 'MustRunAs' + ranges: + - min: {{ .Values.containerSecurityContext.runAsUser }} + max: {{ .Values.containerSecurityContext.runAsUser }} + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: {{ .Values.containerSecurityContext.runAsUser }} + max: {{ .Values.containerSecurityContext.runAsUser }} + volumes: + - 'configMap' + - 'secret' + - 'emptyDir' + - 'persistentVolumeClaim' +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-master-statefulset.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-master-statefulset.yaml new file mode 100644 index 000000000..80dc112a5 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-master-statefulset.yaml @@ -0,0 +1,394 @@ +{{- if or (not .Values.cluster.enabled) (not .Values.sentinel.enabled) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "redis.fullname" . }}-master + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.master.statefulset.labels }} + {{- toYaml .Values.master.statefulset.labels | nindent 4 }} + {{- end }} +{{- if .Values.master.statefulset.annotations }} + annotations: + {{- toYaml .Values.master.statefulset.annotations | nindent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + role: master + serviceName: {{ template "redis.fullname" . }}-headless + template: + metadata: + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + role: master + {{- if .Values.master.podLabels }} + {{- toYaml .Values.master.podLabels | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.podLabels }} + {{- toYaml .Values.metrics.podLabels | nindent 8 }} + {{- end }} + annotations: + checksum/health: {{ include (print $.Template.BasePath "/health-configmap.yaml") . | sha256sum }} + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- if .Values.master.podAnnotations }} + {{- toYaml .Values.master.podAnnotations | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.podAnnotations }} + {{- toYaml .Values.metrics.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- include "redis.imagePullSecrets" . | nindent 6 }} + {{- if .Values.master.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.master.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "redis.serviceAccountName" . }} + {{- if .Values.master.priorityClassName }} + priorityClassName: {{ .Values.master.priorityClassName | quote }} + {{- end }} + {{- with .Values.master.affinity }} + affinity: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- if .Values.master.nodeSelector }} + nodeSelector: {{- toYaml .Values.master.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.master.tolerations }} + tolerations: {{- toYaml .Values.master.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.master.shareProcessNamespace }} + shareProcessNamespace: {{ .Values.master.shareProcessNamespace }} + {{- end }} + {{- if .Values.master.schedulerName }} + schedulerName: {{ .Values.master.schedulerName }} + {{- end }} + containers: + - name: {{ template "redis.name" . }} + image: {{ template "redis.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - /opt/bitnami/scripts/start-scripts/start-master.sh + env: + - name: REDIS_REPLICATION_MODE + value: master + {{- if .Values.usePassword }} + {{- if .Values.usePasswordFile }} + - name: REDIS_PASSWORD_FILE + value: "/opt/bitnami/redis/secrets/redis-password" + {{- else }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + {{- end }} + - name: REDIS_TLS_ENABLED + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: REDIS_TLS_PORT + value: {{ .Values.redisPort | quote }} + - name: REDIS_TLS_AUTH_CLIENTS + value: {{ ternary "yes" "no" .Values.tls.authClients | quote }} + - name: REDIS_TLS_CERT_FILE + value: {{ template "redis.tlsCert" . }} + - name: REDIS_TLS_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_TLS_CA_FILE + value: {{ template "redis.tlsCACert" . }} + {{- if .Values.tls.dhParamsFilename }} + - name: REDIS_TLS_DH_PARAMS_FILE + value: {{ template "redis.tlsDHParams" . }} + {{- end }} + {{- else }} + - name: REDIS_PORT + value: {{ .Values.redisPort | quote }} + {{- end }} + {{- if .Values.master.extraEnvVars }} + {{- include "redis.tplValue" (dict "value" .Values.master.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if or .Values.master.extraEnvVarsCM .Values.master.extraEnvVarsSecret }} + envFrom: + {{- if .Values.master.extraEnvVarsCM }} + - configMapRef: + name: {{ .Values.master.extraEnvVarsCM }} + {{- end }} + {{- if .Values.master.extraEnvVarsSecret }} + - secretRef: + name: {{ .Values.master.extraEnvVarsSecret }} + {{- end }} + {{- end }} + ports: + - name: redis + containerPort: {{ .Values.redisPort }} + {{- if .Values.master.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.master.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.livenessProbe.periodSeconds }} + # One second longer than command timeout should prevent generation of zombie processes. + timeoutSeconds: {{ add1 .Values.master.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.master.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.master.livenessProbe.failureThreshold }} + exec: + command: + - sh + - -c + - /health/ping_liveness_local.sh {{ .Values.master.livenessProbe.timeoutSeconds }} + {{- else if .Values.master.customLivenessProbe }} + livenessProbe: {{- toYaml .Values.master.customLivenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.master.readinessProbe.enabled}} + readinessProbe: + initialDelaySeconds: {{ .Values.master.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.master.readinessProbe.periodSeconds }} + timeoutSeconds: {{ add1 .Values.master.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.master.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.master.readinessProbe.failureThreshold }} + exec: + command: + - sh + - -c + - /health/ping_readiness_local.sh {{ .Values.master.readinessProbe.timeoutSeconds }} + {{- else if .Values.master.customReadinessProbe }} + readinessProbe: {{- toYaml .Values.master.customReadinessProbe | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.master.resources | nindent 12 }} + volumeMounts: + - name: start-scripts + mountPath: /opt/bitnami/scripts/start-scripts + - name: health + mountPath: /health + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /opt/bitnami/redis/secrets/ + {{- end }} + - name: redis-data + mountPath: {{ .Values.master.persistence.path }} + subPath: {{ .Values.master.persistence.subPath }} + - name: config + mountPath: /opt/bitnami/redis/mounted-etc + - name: redis-tmp-conf + mountPath: /opt/bitnami/redis/etc/ + - name: tmp + mountPath: /tmp + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + {{- if .Values.master.extraVolumeMounts }} + {{- include "common.tplvalues.render" ( dict "value" .Values.master.extraVolumeMounts "context" $ ) | nindent 12 }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: metrics + image: {{ template "redis.metrics.image" . }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - | + if [[ -f '/secrets/redis-password' ]]; then + export REDIS_PASSWORD=$(cat /secrets/redis-password) + fi + redis_exporter{{- range $key, $value := .Values.metrics.extraArgs }} --{{ $key }}={{ $value }}{{- end }} + env: + - name: REDIS_ALIAS + value: {{ template "redis.fullname" . }} + {{- if .Values.usePassword }} + - name: REDIS_USER + value: default + {{- if (not .Values.usePasswordFile) }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: REDIS_ADDR + value: rediss://{{ .Values.metrics.redisTargetHost }}:{{ .Values.redisPort }} + {{- if .Values.tls.authClients }} + - name: REDIS_EXPORTER_TLS_CLIENT_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_EXPORTER_TLS_CLIENT_CERT_FILE + value: {{ template "redis.tlsCert" . }} + {{- end }} + - name: REDIS_EXPORTER_TLS_CA_CERT_FILE + value: {{ template "redis.tlsCACert" . }} + {{- end }} + volumeMounts: + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /secrets/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + ports: + - name: metrics + containerPort: 9121 + resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} + {{- $needsVolumePermissions := and .Values.volumePermissions.enabled .Values.master.persistence.enabled .Values.securityContext.enabled .Values.containerSecurityContext.enabled }} + {{- if or $needsVolumePermissions .Values.sysctlImage.enabled }} + initContainers: + {{- if $needsVolumePermissions }} + - name: volume-permissions + image: "{{ template "redis.volumePermissions.image" . }}" + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` {{ .Values.master.persistence.path }} + {{- else }} + chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.master.persistence.path }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto "}} + securityContext: {{- omit .Values.volumePermissions.securityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.securityContext | toYaml | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 10 }} + volumeMounts: + - name: redis-data + mountPath: {{ .Values.master.persistence.path }} + subPath: {{ .Values.master.persistence.subPath }} + {{- end }} + {{- if .Values.sysctlImage.enabled }} + - name: init-sysctl + image: {{ template "redis.sysctl.image" . }} + imagePullPolicy: {{ default "" .Values.sysctlImage.pullPolicy | quote }} + resources: {{- toYaml .Values.sysctlImage.resources | nindent 10 }} + {{- if .Values.sysctlImage.mountHostSys }} + volumeMounts: + - name: host-sys + mountPath: /host-sys + {{- end }} + command: {{- toYaml .Values.sysctlImage.command | nindent 10 }} + securityContext: + privileged: true + runAsUser: 0 + {{- end }} + {{- end }} + volumes: + - name: start-scripts + configMap: + name: {{ include "redis.fullname" . }}-scripts + defaultMode: 0755 + - name: health + configMap: + name: {{ template "redis.fullname" . }}-health + defaultMode: 0755 + {{- if .Values.usePasswordFile }} + - name: redis-password + secret: + secretName: {{ template "redis.secretName" . }} + items: + - key: {{ template "redis.secretPasswordKey" . }} + path: redis-password + {{- end }} + - name: config + configMap: + name: {{ template "redis.fullname" . }} + {{- if not .Values.master.persistence.enabled }} + - name: "redis-data" + emptyDir: {} + {{- else }} + {{- if .Values.persistence.existingClaim }} + - name: "redis-data" + persistentVolumeClaim: + claimName: {{ include "redis.tplValue" (dict "value" .Values.persistence.existingClaim "context" $) }} + {{- end }} + {{- if .Values.master.persistence.volumes }} + {{- toYaml .Values.master.persistence.volumes | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.sysctlImage.mountHostSys }} + - name: host-sys + hostPath: + path: /sys + {{- end }} + - name: redis-tmp-conf + emptyDir: {} + - name: tmp + emptyDir: {} + {{- if .Values.master.extraVolumes }} + {{- include "common.tplvalues.render" ( dict "value" .Values.master.extraVolumes "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: redis-certificates + secret: + secretName: {{ required "A secret containing the certificates for the TLS traffic is required when TLS in enabled" .Values.tls.certificatesSecret }} + defaultMode: 256 + {{- end }} + {{- if and .Values.master.persistence.enabled (not .Values.persistence.existingClaim) (not .Values.master.persistence.volumes) }} + volumeClaimTemplates: + - metadata: + name: redis-data + labels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: master + {{- if .Values.master.statefulset.volumeClaimTemplates.labels }} + {{- toYaml .Values.master.statefulset.volumeClaimTemplates.labels | nindent 10 }} + {{- end }} + {{- if .Values.master.statefulset.volumeClaimTemplates.annotations }} + annotations: + {{- toYaml .Values.master.statefulset.volumeClaimTemplates.annotations | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- range .Values.master.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.master.persistence.size | quote }} + {{ include "redis.master.storageClass" . }} + selector: + {{- if .Values.master.persistence.matchLabels }} + matchLabels: {{- toYaml .Values.master.persistence.matchLabels | nindent 12 }} + {{- end -}} + {{- if .Values.master.persistence.matchExpressions }} + matchExpressions: {{- toYaml .Values.master.persistence.matchExpressions | nindent 12 }} + {{- end -}} + {{- end }} + updateStrategy: + type: {{ .Values.master.statefulset.updateStrategy }} + {{- if .Values.master.statefulset.rollingUpdatePartition }} + {{- if (eq "Recreate" .Values.master.statefulset.updateStrategy) }} + rollingUpdate: null + {{- else }} + rollingUpdate: + partition: {{ .Values.master.statefulset.rollingUpdatePartition }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-master-svc.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-master-svc.yaml new file mode 100644 index 000000000..8bd2f8c1c --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-master-svc.yaml @@ -0,0 +1,43 @@ +{{- if not .Values.sentinel.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "redis.fullname" . }}-master + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.master.service.labels -}} + {{- toYaml .Values.master.service.labels | nindent 4 }} + {{- end -}} +{{- if .Values.master.service.annotations }} + annotations: {{- toYaml .Values.master.service.annotations | nindent 4 }} +{{- end }} +spec: + type: {{ .Values.master.service.type }} + {{ if eq .Values.master.service.type "LoadBalancer" }} + externalTrafficPolicy: {{ .Values.master.service.externalTrafficPolicy }} + {{- end }} + {{- if and (eq .Values.master.service.type "LoadBalancer") .Values.master.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.master.service.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.master.service.type "LoadBalancer") .Values.master.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- with .Values.master.service.loadBalancerSourceRanges }} +{{- toYaml . | nindent 4 }} +{{- end }} + {{- end }} + ports: + - name: tcp-redis + port: {{ .Values.master.service.port }} + targetPort: redis + {{- if .Values.master.service.nodePort }} + nodePort: {{ .Values.master.service.nodePort }} + {{- end }} + selector: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + role: master +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-node-statefulset.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-node-statefulset.yaml new file mode 100644 index 000000000..9ede64175 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-node-statefulset.yaml @@ -0,0 +1,556 @@ +{{- if and .Values.cluster.enabled .Values.sentinel.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "redis.fullname" . }}-node + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.slave.statefulset.labels }} + {{- toYaml .Values.slave.statefulset.labels | nindent 4 }} + {{- end }} +{{- if .Values.slave.statefulset.annotations }} + annotations: + {{- toYaml .Values.slave.statefulset.annotations | nindent 4 }} +{{- end }} +spec: +{{- if .Values.slave.updateStrategy }} + strategy: {{- toYaml .Values.slave.updateStrategy | nindent 4 }} +{{- end }} + replicas: {{ .Values.cluster.slaveCount }} + serviceName: {{ template "redis.fullname" . }}-headless + selector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + role: node + template: + metadata: + labels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + chart: {{ template "redis.chart" . }} + role: node + {{- if .Values.slave.podLabels }} + {{- toYaml .Values.slave.podLabels | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.podLabels }} + {{- toYaml .Values.metrics.podLabels | nindent 8 }} + {{- end }} + annotations: + checksum/health: {{ include (print $.Template.BasePath "/health-configmap.yaml") . | sha256sum }} + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- if .Values.slave.podAnnotations }} + {{- toYaml .Values.slave.podAnnotations | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.podAnnotations }} + {{- toYaml .Values.metrics.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- include "redis.imagePullSecrets" . | nindent 6 }} + {{- if .Values.slave.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.slave.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "redis.serviceAccountName" . }} + {{- if .Values.slave.priorityClassName }} + priorityClassName: "{{ .Values.slave.priorityClassName }}" + {{- end }} + {{- if .Values.slave.nodeSelector }} + nodeSelector: {{- toYaml .Values.slave.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.slave.tolerations }} + tolerations: {{- toYaml .Values.slave.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.slave.schedulerName }} + schedulerName: {{ .Values.slave.schedulerName }} + {{- end }} + {{- if .Values.master.spreadConstraints }} + topologySpreadConstraints: {{- toYaml .Values.master.spreadConstraints | nindent 8 }} + {{- end }} + {{- with .Values.slave.affinity }} + affinity: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + containers: + - name: {{ template "redis.name" . }} + image: {{ template "redis.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - /opt/bitnami/scripts/start-scripts/start-node.sh + env: + - name: REDIS_MASTER_PORT_NUMBER + value: {{ .Values.redisPort | quote }} + {{- if .Values.usePassword }} + {{- if .Values.usePasswordFile }} + - name: REDIS_PASSWORD_FILE + value: "/opt/bitnami/redis/secrets/redis-password" + - name: REDIS_MASTER_PASSWORD_FILE + value: "/opt/bitnami/redis/secrets/redis-password" + {{- else }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + - name: REDIS_MASTER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + {{- end }} + - name: REDIS_TLS_ENABLED + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: REDIS_TLS_PORT + value: {{ .Values.redisPort | quote }} + - name: REDIS_TLS_AUTH_CLIENTS + value: {{ ternary "yes" "no" .Values.tls.authClients | quote }} + - name: REDIS_TLS_CERT_FILE + value: {{ template "redis.tlsCert" . }} + - name: REDIS_TLS_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_TLS_CA_FILE + value: {{ template "redis.tlsCACert" . }} + {{- if .Values.tls.dhParamsFilename }} + - name: REDIS_TLS_DH_PARAMS_FILE + value: {{ template "redis.tlsDHParams" . }} + {{- end }} + {{- else }} + - name: REDIS_PORT + value: {{ .Values.redisPort | quote }} + {{- end }} + - name: REDIS_DATA_DIR + value: {{ .Values.slave.persistence.path }} + {{- if .Values.sentinel.extraEnvVars }} + {{- include "redis.tplValue" (dict "value" .Values.sentinel.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if or .Values.sentinel.extraEnvVarsCM .Values.sentinel.extraEnvVarsSecret }} + envFrom: + {{- if .Values.sentinel.extraEnvVarsCM }} + - configMapRef: + name: {{ .Values.sentinel.extraEnvVarsCM }} + {{- end }} + {{- if .Values.sentinel.extraEnvVarsSecret }} + - secretRef: + name: {{ .Values.sentinel.extraEnvVarsSecret }} + {{- end }} + {{- end }} + ports: + - name: redis + containerPort: {{ .Values.redisPort }} + {{- if .Values.slave.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.slave.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.slave.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.slave.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.slave.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.slave.livenessProbe.failureThreshold}} + exec: + command: + - sh + - -c + {{- if .Values.sentinel.enabled }} + - /health/ping_liveness_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + {{- else }} + - /health/ping_liveness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + {{- end }} + {{- else if .Values.slave.customLivenessProbe }} + livenessProbe: {{- toYaml .Values.slave.customLivenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.slave.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.slave.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.slave.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.slave.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.slave.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.slave.readinessProbe.failureThreshold }} + exec: + command: + - sh + - -c + {{- if .Values.sentinel.enabled }} + - /health/ping_readiness_local.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + {{- else }} + - /health/ping_readiness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + {{- end }} + {{- else if .Values.slave.customReadinessProbe }} + readinessProbe: {{- toYaml .Values.slave.customReadinessProbe | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.slave.resources | nindent 12 }} + volumeMounts: + - name: start-scripts + mountPath: /opt/bitnami/scripts/start-scripts + - name: health + mountPath: /health + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /opt/bitnami/redis/secrets/ + {{- end }} + - name: redis-data + mountPath: {{ .Values.slave.persistence.path }} + subPath: {{ .Values.slave.persistence.subPath }} + - name: config + mountPath: /opt/bitnami/redis/mounted-etc + - name: redis-tmp-conf + mountPath: /opt/bitnami/redis/etc + - name: tmp + mountPath: /tmp + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- include "common.tplvalues.render" ( dict "value" .Values.extraVolumeMounts "context" $ ) | nindent 12 }} + {{- end }} + {{- if and .Values.cluster.enabled .Values.sentinel.enabled }} + - name: sentinel + image: {{ template "sentinel.image" . }} + imagePullPolicy: {{ .Values.sentinel.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - /opt/bitnami/scripts/start-scripts/start-sentinel.sh + env: + {{- if .Values.usePassword }} + {{- if .Values.usePasswordFile }} + - name: REDIS_PASSWORD_FILE + value: "/opt/bitnami/redis/secrets/redis-password" + {{- else }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + {{- end }} + - name: REDIS_SENTINEL_TLS_ENABLED + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: REDIS_SENTINEL_TLS_PORT_NUMBER + value: {{ .Values.sentinel.port | quote }} + - name: REDIS_SENTINEL_TLS_AUTH_CLIENTS + value: {{ ternary "yes" "no" .Values.tls.authClients | quote }} + - name: REDIS_SENTINEL_TLS_CERT_FILE + value: {{ template "redis.tlsCert" . }} + - name: REDIS_SENTINEL_TLS_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_SENTINEL_TLS_CA_FILE + value: {{ template "redis.tlsCACert" . }} + {{- if .Values.tls.dhParamsFilename }} + - name: REDIS_SENTINEL_TLS_DH_PARAMS_FILE + value: {{ template "redis.dhParams" . }} + {{- end }} + {{- else }} + - name: REDIS_SENTINEL_PORT + value: {{ .Values.sentinel.port | quote }} + {{- end }} + ports: + - name: redis-sentinel + containerPort: {{ .Values.sentinel.port }} + {{- if .Values.sentinel.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.sentinel.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.sentinel.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.sentinel.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.sentinel.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.sentinel.livenessProbe.failureThreshold }} + exec: + command: + - sh + - -c + - /health/ping_sentinel.sh {{ .Values.sentinel.livenessProbe.timeoutSeconds }} + {{- else if .Values.sentinel.customLivenessProbe }} + livenessProbe: {{- toYaml .Values.sentinel.customLivenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sentinel.readinessProbe.enabled}} + readinessProbe: + initialDelaySeconds: {{ .Values.sentinel.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.sentinel.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.sentinel.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.sentinel.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.sentinel.readinessProbe.failureThreshold }} + exec: + command: + - sh + - -c + - /health/ping_sentinel.sh {{ .Values.sentinel.livenessProbe.timeoutSeconds }} + {{- else if .Values.sentinel.customReadinessProbe }} + readinessProbe: {{- toYaml .Values.sentinel.customReadinessProbe | nindent 12 }} + {{- end }} + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - /opt/bitnami/scripts/start-scripts/prestop-sentinel.sh + resources: {{- toYaml .Values.sentinel.resources | nindent 12 }} + volumeMounts: + - name: start-scripts + mountPath: /opt/bitnami/scripts/start-scripts + - name: health + mountPath: /health + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /opt/bitnami/redis/secrets/ + {{- end }} + - name: redis-data + mountPath: {{ .Values.slave.persistence.path }} + subPath: {{ .Values.slave.persistence.subPath }} + - name: config + mountPath: /opt/bitnami/redis-sentinel/mounted-etc + - name: sentinel-tmp-conf + mountPath: /opt/bitnami/redis-sentinel/etc + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + {{- if .Values.sentinel.extraVolumeMounts }} + {{- include "common.tplvalues.render" ( dict "value" .Values.sentinel.extraVolumeMounts "context" $ ) | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: metrics + image: {{ template "redis.metrics.image" . }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - | + if [[ -f '/secrets/redis-password' ]]; then + export REDIS_PASSWORD=$(cat /secrets/redis-password) + fi + redis_exporter{{- range $key, $value := .Values.metrics.extraArgs }} --{{ $key }}={{ $value }}{{- end }} + env: + - name: REDIS_ALIAS + value: {{ template "redis.fullname" . }} + {{- if .Values.usePassword }} + - name: REDIS_USER + value: default + {{- if (not .Values.usePasswordFile) }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: REDIS_ADDR + value: rediss://{{ .Values.metrics.redisTargetHost }}:{{ .Values.redisPort }} + {{- if .Values.tls.authClients }} + - name: REDIS_EXPORTER_TLS_CLIENT_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_EXPORTER_TLS_CLIENT_CERT_FILE + value: {{ template "redis.tlsCert" . }} + {{- end }} + - name: REDIS_EXPORTER_TLS_CA_CERT_FILE + value: {{ template "redis.tlsCACert" . }} + {{- end }} + volumeMounts: + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /secrets/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + ports: + - name: metrics + containerPort: 9121 + resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} + {{- if .Values.sentinel.metrics.enabled }} + - name: sentinel-metrics + image: {{ include "sentinel.metrics.image" . }} + imagePullPolicy: {{ .Values.sentinel.metrics.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - redis_sentinel_exporter{{- range $key, $value := .Values.sentinel.metrics.extraArgs }} --{{ $key }}={{ $value }}{{- end }} + env: + {{- if and .Values.sentinel.usePassword (and .Values.usePassword (not .Values.usePasswordFile)) }} + - name: SENTINEL_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis.secretName" . }} + key: {{ include "redis.secretPasswordKey" . }} + {{- end }} + {{- if and .Values.sentinel.usePassword .Values.usePassword .Values.usePasswordFile }} + - name: SENTINEL_PASSWORD_FILE + value: /secrets/redis-password + {{- end }} + volumeMounts: + {{- if and .Values.sentinel.usePassword .Values.usePassword .Values.usePasswordFile }} + - name: redis-password + mountPath: /secrets/ + {{- end }} + ports: + - name: sentinelmetrics + containerPort: 9355 + resources: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.resources "context" $) | nindent 12 }} + {{- end }} + {{- $needsVolumePermissions := and .Values.volumePermissions.enabled .Values.slave.persistence.enabled .Values.securityContext.enabled .Values.containerSecurityContext.enabled }} + {{- if or $needsVolumePermissions .Values.sysctlImage.enabled }} + initContainers: + {{- if $needsVolumePermissions }} + - name: volume-permissions + image: {{ template "redis.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` {{ .Values.slave.persistence.path }} + {{- else }} + chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.slave.persistence.path }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto "}} + securityContext: {{- omit .Values.volumePermissions.securityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.securityContext | toYaml | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + volumeMounts: + - name: redis-data + mountPath: {{ .Values.slave.persistence.path }} + subPath: {{ .Values.slave.persistence.subPath }} + {{- end }} + {{- if .Values.sysctlImage.enabled }} + - name: init-sysctl + image: {{ template "redis.sysctl.image" . }} + imagePullPolicy: {{ default "" .Values.sysctlImage.pullPolicy | quote }} + resources: {{- toYaml .Values.sysctlImage.resources | nindent 12 }} + {{- if .Values.sysctlImage.mountHostSys }} + volumeMounts: + - name: host-sys + mountPath: /host-sys + {{- end }} + command: {{- toYaml .Values.sysctlImage.command | nindent 12 }} + securityContext: + privileged: true + runAsUser: 0 + {{- end }} + {{- end }} + volumes: + - name: start-scripts + configMap: + name: {{ include "redis.fullname" . }}-scripts + defaultMode: 0755 + - name: health + configMap: + name: {{ template "redis.fullname" . }}-health + defaultMode: 0755 + {{- if .Values.usePasswordFile }} + - name: redis-password + secret: + secretName: {{ template "redis.secretName" . }} + items: + - key: {{ template "redis.secretPasswordKey" . }} + path: redis-password + {{- end }} + - name: config + configMap: + name: {{ template "redis.fullname" . }} + {{- if .Values.sysctlImage.mountHostSys }} + - name: host-sys + hostPath: + path: /sys + {{- end }} + - name: sentinel-tmp-conf + emptyDir: {} + - name: redis-tmp-conf + emptyDir: {} + - name: tmp + emptyDir: {} + {{- if .Values.extraVolumes }} + {{- include "common.tplvalues.render" ( dict "value" .Values.extraVolumes "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.sentinel.extraVolumes }} + {{- include "common.tplvalues.render" ( dict "value" .Values.sentinel.extraVolumes "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: redis-certificates + secret: + secretName: {{ required "A secret containing the certificates for the TLS traffic is required when TLS in enabled" .Values.tls.certificatesSecret }} + defaultMode: 256 + {{- end }} + {{- if not .Values.slave.persistence.enabled }} + - name: redis-data + emptyDir: {} + {{- else }} + volumeClaimTemplates: + - metadata: + name: redis-data + labels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: slave + {{- if .Values.slave.statefulset.volumeClaimTemplates }} + {{- if .Values.slave.statefulset.volumeClaimTemplates.labels }} + {{- toYaml .Values.slave.statefulset.volumeClaimTemplates.labels | nindent 10 }} + {{- end }} + {{- if .Values.slave.statefulset.volumeClaimTemplates.annotations }} + annotations: + {{- toYaml .Values.slave.statefulset.volumeClaimTemplates.annotations | nindent 10 }} + {{- end }} + {{- end }} + spec: + accessModes: + {{- range .Values.slave.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.slave.persistence.size | quote }} + {{ include "redis.slave.storageClass" . }} + selector: + {{- if .Values.slave.persistence.matchLabels }} + matchLabels: {{- toYaml .Values.slave.persistence.matchLabels | nindent 12 }} + {{- end -}} + {{- if .Values.slave.persistence.matchExpressions }} + matchExpressions: {{- toYaml .Values.slave.persistence.matchExpressions | nindent 12 }} + {{- end -}} + {{- end }} + updateStrategy: + type: {{ .Values.slave.statefulset.updateStrategy }} + {{- if .Values.slave.statefulset.rollingUpdatePartition }} + {{- if (eq "Recreate" .Values.slave.statefulset.updateStrategy) }} + rollingUpdate: null + {{- else }} + rollingUpdate: + partition: {{ .Values.slave.statefulset.rollingUpdatePartition }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-role.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-role.yaml new file mode 100644 index 000000000..080a7f960 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-role.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.create -}} +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +kind: Role +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: +{{- if .Values.podSecurityPolicy.create }} + - apiGroups: ['{{ template "podSecurityPolicy.apiGroup" . }}'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "redis.fullname" . }}] +{{- end -}} +{{- if .Values.rbac.role.rules }} +{{- toYaml .Values.rbac.role.rules | nindent 2 }} +{{- end -}} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-rolebinding.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-rolebinding.yaml new file mode 100644 index 000000000..835aa0361 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }} +kind: RoleBinding +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "redis.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "redis.serviceAccountName" . }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-serviceaccount.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-serviceaccount.yaml new file mode 100644 index 000000000..081691de6 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "redis.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.serviceAccount.annotations }} + annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-slave-statefulset.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-slave-statefulset.yaml new file mode 100644 index 000000000..778cac5cf --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-slave-statefulset.yaml @@ -0,0 +1,398 @@ +{{- if and .Values.cluster.enabled (not .Values.sentinel.enabled) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "redis.fullname" . }}-slave + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.slave.statefulset.labels }} + {{- toYaml .Values.slave.statefulset.labels | nindent 4 }} + {{- end }} +{{- if .Values.slave.statefulset.annotations }} + annotations: + {{- toYaml .Values.slave.statefulset.annotations | nindent 4 }} +{{- end }} +spec: +{{- if .Values.slave.updateStrategy }} + strategy: {{- toYaml .Values.slave.updateStrategy | nindent 4 }} +{{- end }} + replicas: {{ .Values.cluster.slaveCount }} + serviceName: {{ template "redis.fullname" . }}-headless + selector: + matchLabels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + role: slave + template: + metadata: + labels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + chart: {{ template "redis.chart" . }} + role: slave + {{- if .Values.slave.podLabels }} + {{- toYaml .Values.slave.podLabels | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.podLabels }} + {{- toYaml .Values.metrics.podLabels | nindent 8 }} + {{- end }} + annotations: + checksum/health: {{ include (print $.Template.BasePath "/health-configmap.yaml") . | sha256sum }} + checksum/configmap: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- if .Values.slave.podAnnotations }} + {{- toYaml .Values.slave.podAnnotations | nindent 8 }} + {{- end }} + {{- if and .Values.metrics.enabled .Values.metrics.podAnnotations }} + {{- toYaml .Values.metrics.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- include "redis.imagePullSecrets" . | nindent 6 }} + {{- if .Values.slave.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.slave.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.securityContext.enabled }} + securityContext: {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "redis.serviceAccountName" . }} + {{- if .Values.slave.priorityClassName }} + priorityClassName: {{ .Values.slave.priorityClassName | quote }} + {{- end }} + {{- if .Values.slave.nodeSelector }} + nodeSelector: {{- toYaml .Values.slave.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.slave.tolerations }} + tolerations: {{- toYaml .Values.slave.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.slave.shareProcessNamespace }} + shareProcessNamespace: {{ .Values.slave.shareProcessNamespace }} + {{- end }} + {{- if .Values.slave.schedulerName }} + schedulerName: {{ .Values.slave.schedulerName }} + {{- end }} + {{- if .Values.master.spreadConstraints }} + topologySpreadConstraints: {{- toYaml .Values.master.spreadConstraints | nindent 8 }} + {{- end }} + {{- with .Values.slave.affinity }} + affinity: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + containers: + - name: {{ template "redis.name" . }} + image: {{ template "redis.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - /opt/bitnami/scripts/start-scripts/start-slave.sh + env: + - name: REDIS_REPLICATION_MODE + value: slave + - name: REDIS_MASTER_HOST + value: {{ template "redis.fullname" . }}-master-0.{{ template "redis.fullname" . }}-headless.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + - name: REDIS_MASTER_PORT_NUMBER + value: {{ .Values.redisPort | quote }} + {{- if .Values.usePassword }} + {{- if .Values.usePasswordFile }} + - name: REDIS_PASSWORD_FILE + value: "/opt/bitnami/redis/secrets/redis-password" + - name: REDIS_MASTER_PASSWORD_FILE + value: "/opt/bitnami/redis/secrets/redis-password" + {{- else }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + - name: REDIS_MASTER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- else }} + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + {{- end }} + - name: REDIS_TLS_ENABLED + value: {{ ternary "yes" "no" .Values.tls.enabled | quote }} + {{- if .Values.tls.enabled }} + - name: REDIS_TLS_PORT + value: {{ .Values.redisPort | quote }} + - name: REDIS_TLS_AUTH_CLIENTS + value: {{ ternary "yes" "no" .Values.tls.authClients | quote }} + - name: REDIS_TLS_CERT_FILE + value: {{ template "redis.tlsCert" . }} + - name: REDIS_TLS_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_TLS_CA_FILE + value: {{ template "redis.tlsCACert" . }} + {{- if .Values.tls.dhParamsFilename }} + - name: REDIS_TLS_DH_PARAMS_FILE + value: {{ template "redis.tlsDHParams" . }} + {{- end }} + {{- else }} + - name: REDIS_PORT + value: {{ .Values.redisPort | quote }} + {{- end }} + {{- if .Values.slave.extraEnvVars }} + {{- include "redis.tplValue" (dict "value" .Values.slave.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if or .Values.slave.extraEnvVarsCM .Values.slave.extraEnvVarsSecret }} + envFrom: + {{- if .Values.slave.extraEnvVarsCM }} + - configMapRef: + name: {{ .Values.slave.extraEnvVarsCM }} + {{- end }} + {{- if .Values.slave.extraEnvVarsSecret }} + - secretRef: + name: {{ .Values.slave.extraEnvVarsSecret }} + {{- end }} + {{- end }} + ports: + - name: redis + containerPort: {{ .Values.redisPort }} + {{- if .Values.slave.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.slave.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.slave.livenessProbe.periodSeconds }} + timeoutSeconds: {{ add1 .Values.slave.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.slave.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.slave.livenessProbe.failureThreshold}} + exec: + command: + - sh + - -c + - /health/ping_liveness_local_and_master.sh {{ .Values.slave.livenessProbe.timeoutSeconds }} + {{- else if .Values.slave.customLivenessProbe }} + livenessProbe: {{- toYaml .Values.slave.customLivenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.slave.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.slave.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.slave.readinessProbe.periodSeconds }} + timeoutSeconds: {{ add1 .Values.slave.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.slave.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.slave.readinessProbe.failureThreshold }} + exec: + command: + - sh + - -c + - /health/ping_readiness_local_and_master.sh {{ .Values.slave.readinessProbe.timeoutSeconds }} + {{- else if .Values.slave.customReadinessProbe }} + readinessProbe: {{- toYaml .Values.slave.customReadinessProbe | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.slave.resources | nindent 12 }} + volumeMounts: + - name: start-scripts + mountPath: /opt/bitnami/scripts/start-scripts + - name: health + mountPath: /health + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /opt/bitnami/redis/secrets/ + {{- end }} + - name: redis-data + mountPath: /data + - name: config + mountPath: /opt/bitnami/redis/mounted-etc + - name: redis-tmp-conf + mountPath: /opt/bitnami/redis/etc + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + {{- if .Values.slave.extraVolumeMounts }} + {{- include "common.tplvalues.render" ( dict "value" .Values.slave.extraVolumeMounts "context" $ ) | nindent 12 }} + {{- end }} + {{- if .Values.metrics.enabled }} + - name: metrics + image: {{ template "redis.metrics.image" . }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + command: + - /bin/bash + - -c + - | + if [[ -f '/secrets/redis-password' ]]; then + export REDIS_PASSWORD=$(cat /secrets/redis-password) + fi + redis_exporter{{- range $key, $value := .Values.metrics.extraArgs }} --{{ $key }}={{ $value }}{{- end }} + env: + - name: REDIS_ALIAS + value: {{ template "redis.fullname" . }} + {{- if .Values.usePassword }} + - name: REDIS_USER + value: default + {{- if (not .Values.usePasswordFile) }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis.secretName" . }} + key: {{ template "redis.secretPasswordKey" . }} + {{- end }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: REDIS_ADDR + value: rediss://{{ .Values.metrics.redisTargetHost }}:{{ .Values.redisPort }} + {{- if .Values.tls.authClients }} + - name: REDIS_EXPORTER_TLS_CLIENT_KEY_FILE + value: {{ template "redis.tlsCertKey" . }} + - name: REDIS_EXPORTER_TLS_CLIENT_CERT_FILE + value: {{ template "redis.tlsCert" . }} + {{- end }} + - name: REDIS_EXPORTER_TLS_CA_CERT_FILE + value: {{ template "redis.tlsCACert" . }} + {{- end }} + volumeMounts: + {{- if .Values.usePasswordFile }} + - name: redis-password + mountPath: /secrets/ + {{- end }} + {{- if .Values.tls.enabled }} + - name: redis-certificates + mountPath: /opt/bitnami/redis/certs + readOnly: true + {{- end }} + ports: + - name: metrics + containerPort: 9121 + resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} + {{- $needsVolumePermissions := and .Values.volumePermissions.enabled .Values.slave.persistence.enabled .Values.securityContext.enabled .Values.containerSecurityContext.enabled }} + {{- if or $needsVolumePermissions .Values.sysctlImage.enabled }} + initContainers: + {{- if $needsVolumePermissions }} + - name: volume-permissions + image: {{ template "redis.volumePermissions.image" . }} + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + - -ec + - | + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` {{ .Values.slave.persistence.path }} + {{- else }} + chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.securityContext.fsGroup }} {{ .Values.slave.persistence.path }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.securityContext.runAsUser )) "auto "}} + securityContext: {{- omit .Values.volumePermissions.securityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.securityContext | toYaml | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + volumeMounts: + - name: redis-data + mountPath: {{ .Values.slave.persistence.path }} + subPath: {{ .Values.slave.persistence.subPath }} + {{- end }} + {{- if .Values.sysctlImage.enabled }} + - name: init-sysctl + image: {{ template "redis.sysctl.image" . }} + imagePullPolicy: {{ default "" .Values.sysctlImage.pullPolicy | quote }} + resources: {{- toYaml .Values.sysctlImage.resources | nindent 12 }} + {{- if .Values.sysctlImage.mountHostSys }} + volumeMounts: + - name: host-sys + mountPath: /host-sys + {{- end }} + command: {{- toYaml .Values.sysctlImage.command | nindent 12 }} + securityContext: + privileged: true + runAsUser: 0 + {{- end }} + {{- end }} + volumes: + - name: start-scripts + configMap: + name: {{ include "redis.fullname" . }}-scripts + defaultMode: 0755 + - name: health + configMap: + name: {{ template "redis.fullname" . }}-health + defaultMode: 0755 + {{- if .Values.usePasswordFile }} + - name: redis-password + secret: + secretName: {{ template "redis.secretName" . }} + items: + - key: {{ template "redis.secretPasswordKey" . }} + path: redis-password + {{- end }} + - name: config + configMap: + name: {{ template "redis.fullname" . }} + {{- if .Values.sysctlImage.mountHostSys }} + - name: host-sys + hostPath: + path: /sys + {{- end }} + - name: redis-tmp-conf + emptyDir: {} + {{- if .Values.slave.extraVolumes }} + {{- include "common.tplvalues.render" ( dict "value" .Values.slave.extraVolumes "context" $ ) | nindent 8 }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: redis-certificates + secret: + secretName: {{ required "A secret containing the certificates for the TLS traffic is required when TLS in enabled" .Values.tls.certificatesSecret }} + defaultMode: 256 + {{- end }} + {{- if not .Values.slave.persistence.enabled }} + - name: redis-data + emptyDir: {} + {{- else }} + volumeClaimTemplates: + - metadata: + name: redis-data + labels: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + component: slave + {{- if .Values.slave.statefulset.volumeClaimTemplates }} + {{- if .Values.slave.statefulset.volumeClaimTemplates.labels }} + {{- toYaml .Values.slave.statefulset.volumeClaimTemplates.labels | nindent 10 }} + {{- end }} + {{- if .Values.slave.statefulset.volumeClaimTemplates.annotations }} + annotations: + {{- toYaml .Values.slave.statefulset.volumeClaimTemplates.annotations | nindent 10 }} + {{- end }} + {{- end }} + spec: + accessModes: + {{- range .Values.slave.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.slave.persistence.size | quote }} + {{ include "redis.slave.storageClass" . }} + selector: + {{- if .Values.slave.persistence.matchLabels }} + matchLabels: {{- toYaml .Values.slave.persistence.matchLabels | nindent 12 }} + {{- end -}} + {{- if .Values.slave.persistence.matchExpressions }} + matchExpressions: {{- toYaml .Values.slave.persistence.matchExpressions | nindent 12 }} + {{- end -}} + {{- end }} + updateStrategy: + type: {{ .Values.slave.statefulset.updateStrategy }} + {{- if .Values.slave.statefulset.rollingUpdatePartition }} + {{- if (eq "Recreate" .Values.slave.statefulset.updateStrategy) }} + rollingUpdate: null + {{- else }} + rollingUpdate: + partition: {{ .Values.slave.statefulset.rollingUpdatePartition }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-slave-svc.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-slave-svc.yaml new file mode 100644 index 000000000..a67ebb05e --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-slave-svc.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.cluster.enabled (not .Values.sentinel.enabled) }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "redis.fullname" . }}-slave + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.slave.service.labels -}} + {{- toYaml .Values.slave.service.labels | nindent 4 }} + {{- end -}} +{{- if .Values.slave.service.annotations }} + annotations: {{- toYaml .Values.slave.service.annotations | nindent 4 }} +{{- end }} +spec: + type: {{ .Values.slave.service.type }} + {{ if eq .Values.slave.service.type "LoadBalancer" }} + externalTrafficPolicy: {{ .Values.slave.service.externalTrafficPolicy }} + {{- end }} + {{- if and (eq .Values.slave.service.type "LoadBalancer") .Values.slave.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.slave.service.loadBalancerIP }} + {{- end }} + {{- if and (eq .Values.slave.service.type "LoadBalancer") .Values.slave.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- with .Values.slave.service.loadBalancerSourceRanges }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: tcp-redis + port: {{ .Values.slave.service.port }} + targetPort: redis + {{- if .Values.slave.service.nodePort }} + nodePort: {{ .Values.slave.service.nodePort }} + {{- end }} + selector: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} + role: slave +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-with-sentinel-svc.yaml b/scripts/helm/helm/databases/charts/redis/templates/redis-with-sentinel-svc.yaml new file mode 100644 index 000000000..e1c9073a4 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/redis-with-sentinel-svc.yaml @@ -0,0 +1,43 @@ +{{- if .Values.sentinel.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.sentinel.service.labels }} + {{- toYaml .Values.sentinel.service.labels | nindent 4 }} + {{- end }} +{{- if .Values.sentinel.service.annotations }} + annotations: {{- toYaml .Values.sentinel.service.annotations | nindent 4 }} +{{- end }} +spec: + type: {{ .Values.sentinel.service.type }} + {{ if eq .Values.sentinel.service.type "LoadBalancer" }} + externalTrafficPolicy: {{ .Values.sentinel.service.externalTrafficPolicy }} + {{- end }} + {{ if eq .Values.sentinel.service.type "LoadBalancer" -}} {{ if .Values.sentinel.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.sentinel.service.loadBalancerIP }} + {{ end -}} + {{- end -}} + ports: + - name: tcp-redis + port: {{ .Values.sentinel.service.redisPort }} + targetPort: redis + {{- if .Values.sentinel.service.redisNodePort }} + nodePort: {{ .Values.sentinel.service.redisNodePort }} + {{- end }} + - name: tcp-sentinel + port: {{ .Values.sentinel.service.sentinelPort }} + targetPort: redis-sentinel + {{- if .Values.sentinel.service.sentinelNodePort }} + nodePort: {{ .Values.sentinel.service.sentinelNodePort }} + {{- end }} + selector: + app: {{ template "redis.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/scripts/helm/helm/databases/charts/redis/templates/secret.yaml b/scripts/helm/helm/databases/charts/redis/templates/secret.yaml new file mode 100644 index 000000000..197aa1890 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/templates/secret.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.usePassword (not .Values.existingSecret) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "redis.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "redis.name" . }} + chart: {{ template "redis.chart" . }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + redis-password: {{ include "redis.password" . | b64enc | quote }} +{{- end -}} diff --git a/scripts/helm/helm/databases/charts/redis/values.schema.json b/scripts/helm/helm/databases/charts/redis/values.schema.json new file mode 100644 index 000000000..3188d0c93 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/values.schema.json @@ -0,0 +1,168 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "usePassword": { + "type": "boolean", + "title": "Use password authentication", + "form": true + }, + "password": { + "type": "string", + "title": "Password", + "form": true, + "description": "Defaults to a random 10-character alphanumeric string if not set", + "hidden": { + "value": false, + "path": "usePassword" + } + }, + "cluster": { + "type": "object", + "title": "Cluster Settings", + "form": true, + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable master-slave", + "description": "Enable master-slave architecture" + }, + "slaveCount": { + "type": "integer", + "title": "Slave Replicas", + "form": true, + "hidden": { + "value": false, + "path": "cluster/enabled" + } + } + } + }, + "master": { + "type": "object", + "title": "Master replicas settings", + "form": true, + "properties": { + "persistence": { + "type": "object", + "title": "Persistence for master replicas", + "form": true, + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable persistence", + "description": "Enable persistence using Persistent Volume Claims" + }, + "size": { + "type": "string", + "title": "Persistent Volume Size", + "form": true, + "render": "slider", + "sliderMin": 1, + "sliderMax": 100, + "sliderUnit": "Gi", + "hidden": { + "value": false, + "path": "master/persistence/enabled" + } + }, + "matchLabels": { + "type": "object", + "title": "Persistent Match Labels Selector" + }, + "matchExpressions": { + "type": "object", + "title": "Persistent Match Expressions Selector" + } + } + } + } + }, + "slave": { + "type": "object", + "title": "Slave replicas settings", + "form": true, + "hidden": { + "value": false, + "path": "cluster/enabled" + }, + "properties": { + "persistence": { + "type": "object", + "title": "Persistence for slave replicas", + "form": true, + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable persistence", + "description": "Enable persistence using Persistent Volume Claims" + }, + "size": { + "type": "string", + "title": "Persistent Volume Size", + "form": true, + "render": "slider", + "sliderMin": 1, + "sliderMax": 100, + "sliderUnit": "Gi", + "hidden": { + "value": false, + "path": "slave/persistence/enabled" + } + }, + "matchLabels": { + "type": "object", + "title": "Persistent Match Labels Selector" + }, + "matchExpressions": { + "type": "object", + "title": "Persistent Match Expressions Selector" + } + } + } + } + }, + "volumePermissions": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "form": true, + "title": "Enable Init Containers", + "description": "Use an init container to set required folder permissions on the data volume before mounting it in the final destination" + } + } + }, + "metrics": { + "type": "object", + "form": true, + "title": "Prometheus metrics details", + "properties": { + "enabled": { + "type": "boolean", + "title": "Create Prometheus metrics exporter", + "description": "Create a side-car container to expose Prometheus metrics", + "form": true + }, + "serviceMonitor": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "title": "Create Prometheus Operator ServiceMonitor", + "description": "Create a ServiceMonitor to track metrics using Prometheus Operator", + "form": true, + "hidden": { + "value": false, + "path": "metrics/enabled" + } + } + } + } + } + } + } +} diff --git a/scripts/helm/helm/databases/charts/redis/values.yaml b/scripts/helm/helm/databases/charts/redis/values.yaml new file mode 100644 index 000000000..64e908211 --- /dev/null +++ b/scripts/helm/helm/databases/charts/redis/values.yaml @@ -0,0 +1,1008 @@ +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry and imagePullSecrets +## +global: + # imageRegistry: myRegistryName + # imagePullSecrets: + # - myRegistryKeySecretName + # storageClass: myStorageClass + redis: {} + +## Bitnami Redis(TM) image version +## ref: https://hub.docker.com/r/bitnami/redis/tags/ +## +image: + registry: docker.io + repository: bitnami/redis + ## Bitnami Redis(TM) image tag + ## ref: https://github.com/bitnami/bitnami-docker-redis#supported-tags-and-respective-dockerfile-links + ## + tag: 6.0.12-debian-10-r33 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + +## String to partially override redis.fullname template (will maintain the release name) +## +# nameOverride: + +## String to fully override redis.fullname template +## +# fullnameOverride: + +## Cluster settings +## +cluster: + enabled: true + slaveCount: 2 + +## Use redis sentinel in the redis pod. This will disable the master and slave services and +## create one redis service with ports to the sentinel and the redis instances +## +sentinel: + enabled: false + ## Require password authentication on the sentinel itself + ## ref: https://redis.io/topics/sentinel + ## + usePassword: true + ## Bitnami Redis(TM) Sentintel image version + ## ref: https://hub.docker.com/r/bitnami/redis-sentinel/tags/ + ## + image: + registry: docker.io + repository: bitnami/redis-sentinel + ## Bitnami Redis(TM) image tag + ## ref: https://github.com/bitnami/bitnami-docker-redis-sentinel#supported-tags-and-respective-dockerfile-links + ## + tag: 6.0.12-debian-10-r24 + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + masterSet: mymaster + initialCheckTimeout: 5 + quorum: 2 + downAfterMilliseconds: 20000 + failoverTimeout: 18000 + parallelSyncs: 1 + port: 26379 + + ## Delay seconds when cleaning nodes IPs + ## When starting it will clean the sentinels IP (RESET "*") in all the nodes + ## This is the delay time before sending the command to the next node + ## + cleanDelaySeconds: 5 + + ## Additional Redis(TM) configuration for the sentinel nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Enable or disable static sentinel IDs for each replicas + ## If disabled each sentinel will generate a random id at startup + ## If enabled, each replicas will have a constant ID on each start-up + ## + staticID: false + ## Configure extra options for Redis(TM) Sentinel liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + customLivenessProbe: {} + customReadinessProbe: {} + ## Redis(TM) Sentinel resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + ## Redis(TM) Sentinel Service properties + ## + service: + ## Redis(TM) Sentinel Service type + ## + type: ClusterIP + sentinelPort: 26379 + redisPort: 6379 + + ## External traffic policy (when service type is LoadBalancer) + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # sentinelNodePort: + # redisNodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + loadBalancerIP: + + ## Additional commands to run prior to starting Redis(TM) node with sentinel + ## + preExecCmds: "" + + ## An array to add extra env var to the sentinel node configurations + ## For example: + ## extraEnvVars: + ## - name: name + ## value: value + ## - name: other_name + ## valueFrom: + ## fieldRef: + ## fieldPath: fieldPath + ## + extraEnvVars: [] + + ## ConfigMap with extra env vars: + ## + extraEnvVarsCM: [] + + ## Secret with extra env vars: + ## + extraEnvVarsSecret: [] + + ## Prometheus Exporter / Metrics for Redis Sentinel Exporter + ## + metrics: + enabled: false + + ## Bitnami Redis Sentinel Exporter image + ## ref: https://hub.docker.com/r/bitnami/redis-sentinel-exporter/tags/ + ## + image: + registry: docker.io + repository: bitnami/redis-sentinel-exporter + tag: 1.7.1-debian-10-r105 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + ## Sentinel metrics exporter resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## e.g: + ## limits: + ## cpu: 500m + ## memory: 1Gi + ## + limits: {} + requests: {} + + ## Extra arguments for Sentinel metrics exporter, for example: + ## extraArgs: + ## check-keys: myKey,myOtherKey + # extraArgs: {} + + ## Enable this if you're using https://github.com/coreos/prometheus-operator + ## + serviceMonitor: + enabled: false + ## Specify a namespace if needed + # namespace: monitoring + # fallback to the prometheus default unless specified + # interval: 10s + ## Defaults to what's used if you follow CoreOS [Prometheus Install Instructions](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#tldr) + ## [Prometheus Selector Label](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#exporters) + ## + selector: + prometheus: kube-prometheus + + service: + type: ClusterIP + + ## Metrics port + ## + port: 9355 + ## External traffic policy (when service type is LoadBalancer) + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + + ## Use serviceLoadBalancerIP to request a specific static IP, otherwise leave blank + ## + # loadBalancerIP: + annotations: {} + labels: {} + +## Specifies the Kubernetes Cluster's Domain Name. +## +clusterDomain: cluster.local + +networkPolicy: + ## Specifies whether a NetworkPolicy should be created + ## + enabled: false + + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the port Redis(TM) is listening + ## on. When true, Redis(TM) will accept connections from any source + ## (with the correct destination port). + ## + # allowExternal: true + + ## Allow connections from other namespaces. Just set label for namespace and set label for pods (optional). + ## + ingressNSMatchLabels: {} + ingressNSPodMatchLabels: {} + +serviceAccount: + ## Specifies whether a ServiceAccount should be created + ## + create: false + ## The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the fullname template + ## + name: + ## Add annotations to service account + # annotations: + # iam.gke.io/gcp-service-account: "sa@project.iam.gserviceaccount.com" + +rbac: + ## Specifies whether RBAC resources should be created + ## + create: false + + role: + ## Rules to create. It follows the role specification + # rules: + # - apiGroups: + # - extensions + # resources: + # - podsecuritypolicies + # verbs: + # - use + # resourceNames: + # - gce.unprivileged + rules: [] + +## Redis(TM) pod Security Context +## +securityContext: + enabled: true + fsGroup: 1001 + ## sysctl settings for master and slave pods + ## + ## Uncomment the setting below to increase the net.core.somaxconn value + ## + # sysctls: + # - name: net.core.somaxconn + # value: "10000" + +## Container Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +containerSecurityContext: + enabled: true + runAsUser: 1001 + +## Use password authentication +## +usePassword: true +## Redis(TM) password (both master and slave) +## Defaults to a random 10-character alphanumeric string if not set and usePassword is true +## ref: https://github.com/bitnami/bitnami-docker-redis#setting-the-server-password-on-first-run +## +password: "" +## Use existing secret (ignores previous password) +# existingSecret: +## Password key to be retrieved from Redis(TM) secret +## +# existingSecretPasswordKey: + +## Mount secrets as files instead of environment variables +## +usePasswordFile: false + +## Persist data to a persistent volume (Redis(TM) Master) +## +persistence: + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + ## + existingClaim: + +# Redis(TM) port +redisPort: 6379 + +## +## TLS configuration +## +tls: + # Enable TLS traffic + enabled: false + # + # Whether to require clients to authenticate or not. + authClients: true + # + # Name of the Secret that contains the certificates + certificatesSecret: + # + # Certificate filename + certFilename: + # + # Certificate Key filename + certKeyFilename: + # + # CA Certificate filename + certCAFilename: + # + # File containing DH params (in order to support DH based ciphers) + # dhParamsFilename: + +## +## Redis(TM) Master parameters +## +master: + ## Redis(TM) command arguments + ## + ## Can be used to specify command line arguments, for example: + ## Note `exec` is prepended to command + ## + command: "/run.sh" + ## Additional commands to run prior to starting Redis(TM) + ## + preExecCmds: "" + ## Additional Redis(TM) configuration for the master nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Deployment pod host aliases + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## Redis(TM) additional command line flags + ## + ## Can be used to specify command line flags, for example: + ## extraFlags: + ## - "--maxmemory-policy volatile-ttl" + ## - "--repl-backlog-size 1024mb" + ## + extraFlags: [] + ## Comma-separated list of Redis(TM) commands to disable + ## + ## Can be used to disable Redis(TM) commands for security reasons. + ## Commands will be completely disabled by renaming each to an empty string. + ## ref: https://redis.io/topics/security#disabling-of-specific-commands + ## + disableCommands: + - FLUSHDB + - FLUSHALL + + ## Redis(TM) Master additional pod labels and annotations + ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + ## + podLabels: {} + podAnnotations: {} + + ## Redis(TM) Master resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: + + # Enable shared process namespace in a pod. + # If set to false (default), each container will run in separate namespace, redis will have PID=1. + # If set to true, the /pause will run as init process and will reap any zombie PIDs, + # for example, generated by a custom exec probe running longer than a probe timeoutSeconds. + # Enable this only if customLivenessProbe or customReadinessProbe is used and zombie PIDs are accumulating. + # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/ + shareProcessNamespace: false + ## Configure extra options for Redis(TM) Master liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + + ## Configure custom probes for images other images like + ## rhscl/redis-32-rhel7 rhscl/redis-5-rhel7 + ## Only used if readinessProbe.enabled: false / livenessProbe.enabled: false + ## + # customLivenessProbe: + # tcpSocket: + # port: 6379 + # initialDelaySeconds: 10 + # periodSeconds: 5 + # customReadinessProbe: + # initialDelaySeconds: 30 + # periodSeconds: 10 + # timeoutSeconds: 5 + # exec: + # command: + # - "container-entrypoint" + # - "bash" + # - "-c" + # - "redis-cli set liveness-probe \"`date`\" | grep OK" + customLivenessProbe: {} + customReadinessProbe: {} + + ## Redis(TM) Master Node selectors and tolerations for pod assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature + ## + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + ## Redis(TM) Master pod/node affinity/anti-affinity + ## + affinity: {} + + ## Redis(TM) Master Service properties + ## + service: + ## Redis(TM) Master Service type + ## + type: ClusterIP + port: 6379 + + ## External traffic policy (when service type is LoadBalancer) + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + loadBalancerIP: + # loadBalancerSourceRanges: ["10.0.0.0/8"] + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + ## The path the volume will be mounted at, useful when using different + ## Redis(TM) images. + ## + path: /data + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + ## + subPath: "" + ## redis data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + ## Persistent Volume selectors + ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#selector + ## + matchLabels: {} + matchExpressions: {} + volumes: + # - name: volume_name + # emptyDir: {} + + ## Update strategy, can be set to RollingUpdate or onDelete by default. + ## https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + ## + statefulset: + labels: {} + annotations: {} + updateStrategy: RollingUpdate + ## Partition update strategy + ## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions + # rollingUpdatePartition: + volumeClaimTemplates: + labels: {} + annotations: {} + + ## Redis(TM) Master pod priorityClassName + ## + priorityClassName: null + + ## An array to add extra env vars + ## For example: + ## extraEnvVars: + ## - name: name + ## value: value + ## - name: other_name + ## valueFrom: + ## fieldRef: + ## fieldPath: fieldPath + ## + extraEnvVars: [] + + ## ConfigMap with extra env vars: + ## + extraEnvVarsCM: [] + + ## Secret with extra env vars: + ## + extraEnvVarsSecret: [] + +## +## Redis(TM) Slave properties +## Note: service.type is a mandatory parameter +## The rest of the parameters are either optional or, if undefined, will inherit those declared in Redis(TM) Master +## +slave: + ## Slave Service properties + ## + service: + ## Redis(TM) Slave Service type + ## + type: ClusterIP + ## Redis(TM) port + ## + port: 6379 + + ## External traffic policy (when service type is LoadBalancer) + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + + ## Specify the nodePort value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + # nodePort: + + ## Provide any additional annotations which may be required. This can be used to + ## set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + annotations: {} + labels: {} + loadBalancerIP: + # loadBalancerSourceRanges: ["10.0.0.0/8"] + + ## Redis(TM) slave port + ## + port: 6379 + ## Deployment pod host aliases + ## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ + ## + hostAliases: [] + ## Can be used to specify command line arguments, for example: + ## Note `exec` is prepended to command + ## + command: "/run.sh" + ## Additional commands to run prior to starting Redis(TM) + ## + preExecCmds: "" + ## Additional Redis(TM) configuration for the slave nodes + ## ref: https://redis.io/topics/config + ## + configmap: + ## Redis(TM) extra flags + ## + extraFlags: [] + ## List of Redis(TM) commands to disable + ## + disableCommands: + - FLUSHDB + - FLUSHALL + + ## Redis(TM) Slave pod/node affinity/anti-affinity + ## + affinity: {} + + ## Kubernetes Spread Constraints for pod assignment + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + # - maxSkew: 1 + # topologyKey: node + # whenUnsatisfiable: DoNotSchedule + spreadConstraints: {} + + # Enable shared process namespace in a pod. + # If set to false (default), each container will run in separate namespace, redis will have PID=1. + # If set to true, the /pause will run as init process and will reap any zombie PIDs, + # for example, generated by a custom exec probe running longer than a probe timeoutSeconds. + # Enable this only if customLivenessProbe or customReadinessProbe is used and zombie PIDs are accumulating. + # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/ + shareProcessNamespace: false + ## Configure extra options for Redis(TM) Slave liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + + ## Configure custom probes for images other images like + ## rhscl/redis-32-rhel7 rhscl/redis-5-rhel7 + ## Only used if readinessProbe.enabled: false / livenessProbe.enabled: false + ## + # customLivenessProbe: + # tcpSocket: + # port: 6379 + # initialDelaySeconds: 10 + # periodSeconds: 5 + # customReadinessProbe: + # initialDelaySeconds: 30 + # periodSeconds: 10 + # timeoutSeconds: 5 + # exec: + # command: + # - "container-entrypoint" + # - "bash" + # - "-c" + # - "redis-cli set liveness-probe \"`date`\" | grep OK" + customLivenessProbe: {} + customReadinessProbe: {} + + ## Redis(TM) slave Resource + # resources: + # requests: + # memory: 256Mi + # cpu: 100m + + ## Redis(TM) slave selectors and tolerations for pod assignment + # nodeSelector: {"beta.kubernetes.io/arch": "amd64"} + # tolerations: [] + + ## Use an alternate scheduler, e.g. "stork". + ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ + ## + # schedulerName: + + ## Redis(TM) slave pod Annotation and Labels + ## + podLabels: {} + podAnnotations: {} + + ## Redis(TM) slave pod priorityClassName + priorityClassName: null + + ## Enable persistence using Persistent Volume Claims + ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + enabled: true + ## The path the volume will be mounted at, useful when using different + ## Redis(TM) images. + ## + path: /data + ## The subdirectory of the volume to mount to, useful in dev environments + ## and one PV for multiple services. + ## + subPath: "" + ## redis data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + accessModes: + - ReadWriteOnce + size: 8Gi + ## Persistent Volume selectors + ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#selector + ## + matchLabels: {} + matchExpressions: {} + + ## Update strategy, can be set to RollingUpdate or onDelete by default. + ## https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + ## + statefulset: + labels: {} + annotations: {} + updateStrategy: RollingUpdate + ## Partition update strategy + ## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions + # rollingUpdatePartition: + volumeClaimTemplates: + labels: {} + annotations: {} + + ## An array to add extra env vars + ## For example: + ## extraEnvVars: + ## - name: name + ## value: value + ## - name: other_name + ## valueFrom: + ## fieldRef: + ## fieldPath: fieldPath + ## + extraEnvVars: [] + + ## ConfigMap with extra env vars: + ## + extraEnvVarsCM: [] + + ## Secret with extra env vars: + ## + extraEnvVarsSecret: [] + +## Prometheus Exporter / Metrics +## +metrics: + enabled: false + + image: + registry: docker.io + repository: bitnami/redis-exporter + tag: 1.20.0-debian-10-r12 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + # A way to specify an alternative redis hostname, if you set a local endpoint in hostAliases for example + # Useful for certificate CN/SAN matching + redisTargetHost: "localhost" + + ## Metrics exporter resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + # resources: {} + + ## Extra arguments for Metrics exporter, for example: + ## extraArgs: + ## check-keys: myKey,myOtherKey + # extraArgs: {} + + ## Metrics exporter pod Annotation and Labels + ## + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9121" + # podLabels: {} + + # Enable this if you're using https://github.com/coreos/prometheus-operator + serviceMonitor: + enabled: false + ## Specify a namespace if needed + # namespace: monitoring + # fallback to the prometheus default unless specified + # interval: 10s + ## Defaults to what's used if you follow CoreOS [Prometheus Install Instructions](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#tldr) + ## [Prometheus Selector Label](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-operator-1) + ## [Kube Prometheus Selector Label](https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#exporters) + ## + selector: + prometheus: kube-prometheus + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## Value is evalued as a template + ## + relabelings: [] + + ## MetricRelabelConfigs to apply to samples before ingestion + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## Value is evalued as a template + ## + metricRelabelings: [] + # - sourceLabels: + # - "__name__" + # targetLabel: "__name__" + # action: replace + # regex: '(.*)' + # replacement: 'example_prefix_$1' + + ## Custom PrometheusRule to be defined + ## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart + ## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions + ## + prometheusRule: + enabled: false + additionalLabels: {} + namespace: "" + ## Redis(TM) prometheus rules + ## These are just examples rules, please adapt them to your needs. + ## Make sure to constraint the rules to the current redis service. + # rules: + # - alert: RedisDown + # expr: redis_up{service="{{ template "redis.fullname" . }}-metrics"} == 0 + # for: 2m + # labels: + # severity: error + # annotations: + # summary: Redis(TM) instance {{ "{{ $labels.instance }}" }} down + # description: Redis(TM) instance {{ "{{ $labels.instance }}" }} is down + # - alert: RedisMemoryHigh + # expr: > + # redis_memory_used_bytes{service="{{ template "redis.fullname" . }}-metrics"} * 100 + # / + # redis_memory_max_bytes{service="{{ template "redis.fullname" . }}-metrics"} + # > 90 + # for: 2m + # labels: + # severity: error + # annotations: + # summary: Redis(TM) instance {{ "{{ $labels.instance }}" }} is using too much memory + # description: | + # Redis(TM) instance {{ "{{ $labels.instance }}" }} is using {{ "{{ $value }}" }}% of its available memory. + # - alert: RedisKeyEviction + # expr: | + # increase(redis_evicted_keys_total{service="{{ template "redis.fullname" . }}-metrics"}[5m]) > 0 + # for: 1s + # labels: + # severity: error + # annotations: + # summary: Redis(TM) instance {{ "{{ $labels.instance }}" }} has evicted keys + # description: | + # Redis(TM) instance {{ "{{ $labels.instance }}" }} has evicted {{ "{{ $value }}" }} keys in the last 5 minutes. + rules: [] + + ## Metrics exporter pod priorityClassName + priorityClassName: null + service: + type: ClusterIP + + ## External traffic policy (when service type is LoadBalancer) + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + + ## Use serviceLoadBalancerIP to request a specific static IP, + ## otherwise leave blank + # loadBalancerIP: + annotations: {} + labels: {} + +## +## Init containers parameters: +## volumePermissions: Change the owner of the persist volume mountpoint to RunAsUser:fsGroup +## +volumePermissions: + enabled: false + image: + registry: docker.io + repository: bitnami/bitnami-shell + tag: "10" + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + resources: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + + ## Init container Security Context + ## Note: the chown of the data folder is done to containerSecurityContext.runAsUser + ## and not the below volumePermissions.securityContext.runAsUser + ## When runAsUser is set to special value "auto", init container will try to chwon the + ## data folder to autodetermined user&group, using commands: `id -u`:`id -G | cut -d" " -f2` + ## "auto" is especially useful for OpenShift which has scc with dynamic userids (and 0 is not allowed). + ## You may want to use this volumePermissions.securityContext.runAsUser="auto" in combination with + ## podSecurityContext.enabled=false,containerSecurityContext.enabled=false + ## + securityContext: + runAsUser: 0 + +## Redis(TM) config file +## ref: https://redis.io/topics/config +## +configmap: |- + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly yes + # Disable RDB persistence, AOF persistence already enabled. + save "" + +## Sysctl InitContainer +## used to perform sysctl operation to modify Kernel settings (needed sometimes to avoid warnings) +## +sysctlImage: + enabled: false + command: [] + registry: docker.io + repository: bitnami/bitnami-shell + tag: "10" + pullPolicy: Always + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + mountHostSys: false + resources: {} + # resources: + # requests: + # memory: 128Mi + # cpu: 100m + +## PodSecurityPolicy configuration +## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + ## Specifies whether a PodSecurityPolicy should be created + ## + create: false + +## Define a disruption budget +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ +## +podDisruptionBudget: + enabled: false + minAvailable: 1 + # maxUnavailable: 1 diff --git a/scripts/helm/helm/databases/templates/NOTES.txt b/scripts/helm/helm/databases/templates/NOTES.txt new file mode 100644 index 000000000..0a725d663 --- /dev/null +++ b/scripts/helm/helm/databases/templates/NOTES.txt @@ -0,0 +1,5 @@ +To get minio credentials run: + + echo "AccessKey: `kubectl get secret --namespace {{ .Release.Namespace }} minio -o jsonpath="{.data.access-key}" | base64 --decode`" + echo "SecretKey: `kubectl get secret --namespace {{ .Release.Namespace }} minio -o jsonpath="{.data.secret-key}" | base64 --decode`" + diff --git a/scripts/helm/helm/databases/templates/_helpers.tpl b/scripts/helm/helm/databases/templates/_helpers.tpl new file mode 100644 index 000000000..066c03a17 --- /dev/null +++ b/scripts/helm/helm/databases/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "databases.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "databases.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "databases.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "databases.labels" -}} +helm.sh/chart: {{ include "databases.chart" . }} +{{ include "databases.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "databases.selectorLabels" -}} +app.kubernetes.io/name: {{ include "databases.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "databases.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "databases.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/databases/values.yaml b/scripts/helm/helm/databases/values.yaml new file mode 100644 index 000000000..13b52cd60 --- /dev/null +++ b/scripts/helm/helm/databases/values.yaml @@ -0,0 +1,140 @@ +# Default values for databases. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + + + +## Child charts +redis: + enabled: true + fullnameOverride: redis + usePassword: false + cluster: + enabled: false + redis: + resources: + limits: + cpu: 250m + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + +postgresql: + # postgresqlPassword: asayerPostgres + fullnameOverride: postgresql + image: + tag: 13.3.0-debian-10-r53 + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 250m + memory: 256Mi + +minio: + # global: + # minio: + # accessKey: "{{ minio_access_key }}" + # secretKey: "{{ minio_secret_key }}" + fullnameOverride: minio + resources: + limits: + cpu: 256m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +kafka: + fullnameOverride: kafka + enabled: false + + +# Enterprise dbs +clickhouse: + enabled: false + +postgreql: + enabled: true + diff --git a/scripts/helm/helm/init.sh b/scripts/helm/helm/init.sh new file mode 100644 index 000000000..3e6d2a96b --- /dev/null +++ b/scripts/helm/helm/init.sh @@ -0,0 +1,104 @@ +#/bin/bash + +# --- helper functions for logs --- +info() +{ + echo '[INFO] ' "$@" +} +warn() +{ + echo '[WARN] ' "$@" >&2 +} +fatal() +{ + echo '[ERROR] ' "$@" >&2 + exit 1 +} + +version=${OPENREPLAY_VERSION:-'v1.3.6'} + +# Check whether script is root user +[[ $(id -u) -eq 0 ]] || { + fatal "Please run the script as root user" + exit 100 +} + +# Installing k3s +curl -sL https://get.k3s.io | sudo K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION='v1.19.5+k3s2' INSTALL_K3S_EXEC="--no-deploy=traefik" sh - +mkdir ~/.kube +cp /etc/rancher/k3s/k3s.yaml ~/.kube/config +chmod 0644 ~/.kube/config + +## installing kubectl +which kubectl &> /dev/null || { + info "kubectl not installed. Installing it..." + sudo curl -SsL https://dl.k8s.io/release/v1.20.0/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl ; sudo chmod +x /usr/local/bin/kubectl +} + +## installing stern +which stern &> /dev/null || { + info "stern not installed. installing..." + sudo curl -SsL https://github.com/derdanne/stern/releases/download/2.1.16/stern_linux_amd64 -o /usr/local/bin/stern ; sudo chmod +x /usr/local/bin/stern +} + +## installing k9s +which k9s &> /dev/null || { + info "k9s not installed. Installing it..." + sudo curl -SsL https://github.com/derailed/k9s/releases/download/v0.24.2/k9s_Linux_x86_64.tar.gz -o /tmp/k9s.tar.gz + cd /tmp + tar -xf k9s.tar.gz + sudo mv k9s /usr/local/bin/k9s + sudo chmod +x /usr/local/bin/k9s + cd - +} + +## installing helm +which helm &> /dev/null +if [[ $? -ne 0 ]]; then + info "helm not installed. Installing it..." + curl -ssl https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz -o /tmp/helm.tar.gz + tar -xf /tmp/helm.tar.gz + chmod +x linux-amd64/helm + sudo cp linux-amd64/helm /usr/local/bin/helm + rm -rf linux-amd64/helm /tmp/helm.tar.gz +fi + +## Installing openssl +apt update &> /dev/null +apt install openssl -y &> /dev/null + +randomPass() { + openssl rand -hex 10 +} + +## Prepping the infra + +## Don't override existing variables file. +[[ -f vars.yaml ]] && { + warn "Existing Variables file. Not overriding." +}|| { + +[[ -z $DOMAIN_NAME ]] && { +fatal "DOMAIN_NAME variable is empty. Rerun the script" +info "curl -sL get.openreplay.com | sudo DOMAIN_NAME=openerplay.mycomp.org bash - " +exit 101 +} + +info " Downloading vars file" +curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${version}/scripts/helm/helm/vars.yaml + +info "Creating dynamic passwords" +sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"$(randomPass)\"/g" vars.yaml +sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"$(randomPass)\"/g" vars.yaml +sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"$(randomPass)\"/g" vars.yaml +sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"$(randomPass)\"/g" vars.yaml +sed -i "s/domainName: .*/domainName: \"${DOMAIN_NAME}\"/g" vars.yaml + +} + +## Installing OpenReplay +info "Installing databases" +helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./vars.yaml --atomic +info "Installing application" +helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./vars.yaml --atomic + diff --git a/scripts/helm/helm/openreplay/.helmignore b/scripts/helm/helm/openreplay/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/Chart.yaml b/scripts/helm/helm/openreplay/Chart.yaml new file mode 100644 index 000000000..70783afea --- /dev/null +++ b/scripts/helm/helm/openreplay/Chart.yaml @@ -0,0 +1,25 @@ +apiVersion: v2 +name: openreplay +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +# Ref: https://github.com/helm/helm/issues/7858#issuecomment-608114589 +AppVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/alerts/.helmignore b/scripts/helm/helm/openreplay/charts/alerts/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/alerts/Chart.yaml b/scripts/helm/helm/openreplay/charts/alerts/Chart.yaml new file mode 100644 index 000000000..101e24ac1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: alerts +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/alerts/templates/NOTES.txt new file mode 100644 index 000000000..4aded5587 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "alerts.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "alerts.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "alerts.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "alerts.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/alerts/templates/_helpers.tpl new file mode 100644 index 000000000..35ad32196 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "alerts.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "alerts.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "alerts.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "alerts.labels" -}} +helm.sh/chart: {{ include "alerts.chart" . }} +{{ include "alerts.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "alerts.selectorLabels" -}} +app.kubernetes.io/name: {{ include "alerts.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "alerts.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "alerts.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/alerts/templates/deployment.yaml new file mode 100644 index 000000000..3639ae413 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "alerts.fullname" . }} + labels: + {{- include "alerts.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "alerts.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "alerts.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "alerts.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: POSTGRES_STRING + value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:{{ .Values.global.postgresql.postgresqlPassword }}@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/alerts/templates/hpa.yaml new file mode 100644 index 000000000..b25fef03d --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "alerts.fullname" . }} + labels: + {{- include "alerts.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "alerts.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/alerts/templates/ingress.yaml new file mode 100644 index 000000000..497e2ec3c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "alerts.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "alerts.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/service.yaml b/scripts/helm/helm/openreplay/charts/alerts/templates/service.yaml new file mode 100644 index 000000000..a002cfd1b --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "alerts.fullname" . }} + labels: + {{- include "alerts.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "alerts.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/alerts/templates/serviceaccount.yaml new file mode 100644 index 000000000..044787236 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "alerts.serviceAccountName" . }} + labels: + {{- include "alerts.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/alerts/templates/tests/test-connection.yaml new file mode 100644 index 000000000..05b0f0496 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "alerts.fullname" . }}-test-connection" + labels: + {{- include "alerts.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "alerts.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/alerts/values.yaml b/scripts/helm/helm/openreplay/charts/alerts/values.yaml new file mode 100644 index 000000000..9a353b057 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/alerts/values.yaml @@ -0,0 +1,87 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/alerts + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "alerts" +fullnameOverride: "alerts" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + ALERT_NOTIFICATION_STRING: http://chalice-openreplay.app.svc.cluster.local:8000/alerts/notifications + CLICKHOUSE_STRING: 'tcp://clickhouse.db.svc.cluster.local:9000/default' + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/assets/.helmignore b/scripts/helm/helm/openreplay/charts/assets/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/assets/Chart.yaml b/scripts/helm/helm/openreplay/charts/assets/Chart.yaml new file mode 100644 index 000000000..924e9d634 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: assets +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/assets/templates/NOTES.txt new file mode 100644 index 000000000..1758d5c64 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "assets.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "assets.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "assets.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "assets.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/assets/templates/_helpers.tpl new file mode 100644 index 000000000..6d684eed5 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "assets.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "assets.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "assets.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "assets.labels" -}} +helm.sh/chart: {{ include "assets.chart" . }} +{{ include "assets.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "assets.selectorLabels" -}} +app.kubernetes.io/name: {{ include "assets.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "assets.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "assets.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/assets/templates/deployment.yaml new file mode 100644 index 000000000..5df3ee4a8 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/deployment.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "assets.fullname" . }} + labels: + {{- include "assets.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "assets.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "assets.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "assets.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AWS_ACCESS_KEY_ID + value: '{{ .Values.global.s3.accessKey }}' + - name: AWS_SECRET_ACCESS_KEY + value: '{{ .Values.global.s3.secretKey }}' + - name: S3_BUCKET_ASSETS + value: '{{ .Values.global.s3.assetsBucket }}' + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: AWS_ENDPOINT + value: '{{ .Values.global.s3.endpoint }}' + - name: AWS_REGION + value: '{{ .Values.global.s3.region }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + # Ref: https://stackoverflow.com/questions/53634583/go-template-split-string-by-delimiter + # We need https://bucketname.s3endpoint + - name: ASSETS_ORIGIN + value: '{{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/assets/templates/hpa.yaml new file mode 100644 index 000000000..7b271a20e --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "assets.fullname" . }} + labels: + {{- include "assets.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "assets.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/assets/templates/ingress.yaml new file mode 100644 index 000000000..62c421a9a --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "assets.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "assets.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/service.yaml b/scripts/helm/helm/openreplay/charts/assets/templates/service.yaml new file mode 100644 index 000000000..5e613c67e --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "assets.fullname" . }} + labels: + {{- include "assets.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "assets.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/assets/templates/serviceaccount.yaml new file mode 100644 index 000000000..90c3f5319 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "assets.serviceAccountName" . }} + labels: + {{- include "assets.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/assets/templates/tests/test-connection.yaml new file mode 100644 index 000000000..ce67efb1c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "assets.fullname" . }}-test-connection" + labels: + {{- include "assets.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "assets.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/assets/values.yaml b/scripts/helm/helm/openreplay/charts/assets/values.yaml new file mode 100644 index 000000000..875d0450a --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/assets/values.yaml @@ -0,0 +1,84 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/assets + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "assets" +fullnameOverride: "assets" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/chalice/.helmignore b/scripts/helm/helm/openreplay/charts/chalice/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/chalice/Chart.yaml b/scripts/helm/helm/openreplay/charts/chalice/Chart.yaml new file mode 100644 index 000000000..4516f0b9b --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: chalice +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/chalice/templates/NOTES.txt new file mode 100644 index 000000000..ecc5a589c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "chalice.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "chalice.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "chalice.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "chalice.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/chalice/templates/_helpers.tpl new file mode 100644 index 000000000..27c1ff9f3 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "chalice.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "chalice.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "chalice.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "chalice.labels" -}} +helm.sh/chart: {{ include "chalice.chart" . }} +{{ include "chalice.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "chalice.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chalice.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "chalice.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "chalice.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/chalice/templates/deployment.yaml new file mode 100644 index 000000000..a092c29aa --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/deployment.yaml @@ -0,0 +1,92 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "chalice.fullname" . }} + labels: + {{- include "chalice.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "chalice.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "chalice.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "chalice.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: pg_host + value: '{{ .Values.global.postgresql.postgresqlHost }}' + - name: pg_port + value: "5432" + - name: pg_dbname + value: "{{ .Values.global.postgresql.postgresqlDatabase }}" + - name: pg_user + value: '{{ .Values.global.postgresql.postgresqlUser }}' + - name: pg_password + value: '{{ .Values.global.postgresql.postgresqlPassword }}' + - name: S3_HOST + {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} + value: 'https://{{ .Values.global.domainName }}' + {{- else}} + value: '{{ .Values.global.s3.endpoint }}' + {{- end}} + - name: S3_KEY + value: '{{ .Values.global.s3.accessKey }}' + - name: S3_SECRET + value: '{{ .Values.global.s3.secretKey }}' + - name: AWS_DEFAULT_REGION + value: '{{ .Values.global.s3.region }}' + - name: sessions_region + value: '{{ .Values.global.s3.region }}' + - name: sessions_bucket + value: '{{ .Values.global.s3.recordings_bucket }}' + - name: sourcemaps_bucket + value: '{{ .Values.global.s3.sourcemaps_bucket }}' + - name: js_cache_bucket + value: '{{ .Values.global.s3.assetsBucket }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/chalice/templates/hpa.yaml new file mode 100644 index 000000000..dc1cb12c1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "chalice.fullname" . }} + labels: + {{- include "chalice.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "chalice.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/chalice/templates/ingress.yaml new file mode 100644 index 000000000..8f680e949 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "chalice.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "chalice.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/service.yaml b/scripts/helm/helm/openreplay/charts/chalice/templates/service.yaml new file mode 100644 index 000000000..02b01d1ab --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "chalice.fullname" . }} + labels: + {{- include "chalice.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "chalice.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/chalice/templates/serviceaccount.yaml new file mode 100644 index 000000000..9af2e3ac4 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "chalice.serviceAccountName" . }} + labels: + {{- include "chalice.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/chalice/templates/tests/test-connection.yaml new file mode 100644 index 000000000..4e76f6cd3 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "chalice.fullname" . }}-test-connection" + labels: + {{- include "chalice.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "chalice.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/chalice/values.yaml b/scripts/helm/helm/openreplay/charts/chalice/values.yaml new file mode 100644 index 000000000..bb8139ede --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/chalice/values.yaml @@ -0,0 +1,116 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/chalice + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "chalice" +fullnameOverride: "chalice" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 8000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + ch_host: clickhouse.db.svc.cluster.local + ch_port: 9000 + captcha_server: '' + captcha_key: '' + async_Token: '' + EMAIL_HOST: '' + EMAIL_PORT: '587' + EMAIL_USER: '' + EMAIL_PASSWORD: '' + EMAIL_USE_TLS: 'true' + EMAIL_USE_SSL: 'false' + EMAIL_SSL_KEY: '' + EMAIL_SSL_CERT: '' + EMAIL_FROM: OpenReplay + SITE_URL: '' + announcement_url: '' + jwt_secret: "SetARandomStringHere" + jwt_algorithm: HS512 + jwt_exp_delta_seconds: '2592000' + # Enable logging for python app + # Ref: https://stackoverflow.com/questions/43969743/logs-in-kubernetes-pod-not-showing-up + PYTHONUNBUFFERED: '0' + version_number: '1.3.6' + SAML2_MD_URL: '' + idp_entityId: '' + idp_sso_url: '' + idp_x509cert: '' + idp_sls_url: '' + idp_name: '' + assist_secret: '' + iceServers: '' + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/db/.helmignore b/scripts/helm/helm/openreplay/charts/db/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/db/Chart.yaml b/scripts/helm/helm/openreplay/charts/db/Chart.yaml new file mode 100644 index 000000000..816ccccb9 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: db +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/db/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/db/templates/NOTES.txt new file mode 100644 index 000000000..b066541cf --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "db.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "db.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "db.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "db.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/db/templates/_helpers.tpl new file mode 100644 index 000000000..8df84e9e2 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "db.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "db.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "db.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "db.labels" -}} +helm.sh/chart: {{ include "db.chart" . }} +{{ include "db.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "db.selectorLabels" -}} +app.kubernetes.io/name: {{ include "db.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "db.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "db.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/db/templates/deployment.yaml new file mode 100644 index 000000000..fd879d92f --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/deployment.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "db.fullname" . }} + labels: + {{- include "db.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "db.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "db.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "db.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + - name: POSTGRES_STRING + value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:{{ .Values.global.postgresql.postgresqlPassword }}@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/db/templates/hpa.yaml new file mode 100644 index 000000000..8dbac030f --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "db.fullname" . }} + labels: + {{- include "db.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "db.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/db/templates/ingress.yaml new file mode 100644 index 000000000..1cbf3640e --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "db.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "db.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/service.yaml b/scripts/helm/helm/openreplay/charts/db/templates/service.yaml new file mode 100644 index 000000000..e0fcac464 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "db.fullname" . }} + labels: + {{- include "db.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "db.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/db/templates/serviceaccount.yaml new file mode 100644 index 000000000..80decdb84 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "db.serviceAccountName" . }} + labels: + {{- include "db.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/db/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/db/templates/tests/test-connection.yaml new file mode 100644 index 000000000..9dd1bdac9 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "db.fullname" . }}-test-connection" + labels: + {{- include "db.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "db.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/db/values.yaml b/scripts/helm/helm/openreplay/charts/db/values.yaml new file mode 100644 index 000000000..77acb5a80 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/db/values.yaml @@ -0,0 +1,85 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/db + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "db" +fullnameOverride: "db" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: {} + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/ender/.helmignore b/scripts/helm/helm/openreplay/charts/ender/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/ender/Chart.yaml b/scripts/helm/helm/openreplay/charts/ender/Chart.yaml new file mode 100644 index 000000000..0e3fadd03 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: ender +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/ender/templates/NOTES.txt new file mode 100644 index 000000000..13aaa8365 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ender.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ender.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ender.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ender.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/ender/templates/_helpers.tpl new file mode 100644 index 000000000..1d0dc291c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ender.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ender.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ender.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ender.labels" -}} +helm.sh/chart: {{ include "ender.chart" . }} +{{ include "ender.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ender.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ender.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ender.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ender.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/ender/templates/deployment.yaml new file mode 100644 index 000000000..bbdb35c00 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ender.fullname" . }} + labels: + {{- include "ender.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "ender.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "ender.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "ender.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/ender/templates/hpa.yaml new file mode 100644 index 000000000..62138535e --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "ender.fullname" . }} + labels: + {{- include "ender.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "ender.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/ender/templates/ingress.yaml new file mode 100644 index 000000000..7f6cc557b --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "ender.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "ender.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/service.yaml b/scripts/helm/helm/openreplay/charts/ender/templates/service.yaml new file mode 100644 index 000000000..be0b79e9e --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ender.fullname" . }} + labels: + {{- include "ender.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "ender.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/ender/templates/serviceaccount.yaml new file mode 100644 index 000000000..f85c1d07b --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ender.serviceAccountName" . }} + labels: + {{- include "ender.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/ender/templates/tests/test-connection.yaml new file mode 100644 index 000000000..8fa7e3b55 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "ender.fullname" . }}-test-connection" + labels: + {{- include "ender.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "ender.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/ender/values.yaml b/scripts/helm/helm/openreplay/charts/ender/values.yaml new file mode 100644 index 000000000..d5a7c52b1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/ender/values.yaml @@ -0,0 +1,85 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/ender + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "ender" +fullnameOverride: "ender" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: {} + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/http/.helmignore b/scripts/helm/helm/openreplay/charts/http/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/http/Chart.yaml b/scripts/helm/helm/openreplay/charts/http/Chart.yaml new file mode 100644 index 000000000..54a567e21 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: http +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/http/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/http/templates/NOTES.txt new file mode 100644 index 000000000..f3b24def1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "http.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "http.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "http.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "http.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/http/templates/_helpers.tpl new file mode 100644 index 000000000..c695edd7e --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "http.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "http.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "http.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "http.labels" -}} +helm.sh/chart: {{ include "http.chart" . }} +{{ include "http.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "http.selectorLabels" -}} +app.kubernetes.io/name: {{ include "http.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "http.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "http.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/http/templates/deployment.yaml new file mode 100644 index 000000000..7de2a80c5 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/deployment.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "http.fullname" . }} + labels: + {{- include "http.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "http.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "http.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "http.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AWS_ACCESS_KEY_ID + value: '{{ .Values.global.s3.accessKey }}' + - name: AWS_SECRET_ACCESS_KEY + value: '{{ .Values.global.s3.secretKey }}' + - name: AWS_REGION + value: '{{ .Values.global.s3.region }}' + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + - name: POSTGRES_STRING + value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:{{ .Values.global.postgresql.postgresqlPassword }}@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + - name: ASSETS_ORIGIN + value: '{{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/http/templates/hpa.yaml new file mode 100644 index 000000000..8e88b2807 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "http.fullname" . }} + labels: + {{- include "http.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "http.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/http/templates/ingress.yaml new file mode 100644 index 000000000..51a20e2a6 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "http.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "http.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/service.yaml b/scripts/helm/helm/openreplay/charts/http/templates/service.yaml new file mode 100644 index 000000000..f33db7394 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "http.fullname" . }} + labels: + {{- include "http.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "http.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/http/templates/serviceaccount.yaml new file mode 100644 index 000000000..7c55e5954 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "http.serviceAccountName" . }} + labels: + {{- include "http.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/http/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/http/templates/tests/test-connection.yaml new file mode 100644 index 000000000..0f29e5a06 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "http.fullname" . }}-test-connection" + labels: + {{- include "http.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "http.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/http/values.yaml b/scripts/helm/helm/openreplay/charts/http/values.yaml new file mode 100644 index 000000000..e841b4cac --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/http/values.yaml @@ -0,0 +1,89 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/http + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "http" +fullnameOverride: "http" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + TOKEN_SECRET: secret_token_string # TODO: generate on buld + S3_BUCKET_IOS_IMAGES: sessions-mobile-assets + CACHE_ASSETS: false + HTTP_PORT: 80 + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/integrations/.helmignore b/scripts/helm/helm/openreplay/charts/integrations/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/integrations/Chart.yaml b/scripts/helm/helm/openreplay/charts/integrations/Chart.yaml new file mode 100644 index 000000000..48fffed45 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: integrations +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/integrations/templates/NOTES.txt new file mode 100644 index 000000000..b7af1aaf2 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "integrations.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "integrations.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "integrations.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "integrations.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/integrations/templates/_helpers.tpl new file mode 100644 index 000000000..cdeef0f54 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "integrations.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "integrations.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "integrations.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "integrations.labels" -}} +helm.sh/chart: {{ include "integrations.chart" . }} +{{ include "integrations.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "integrations.selectorLabels" -}} +app.kubernetes.io/name: {{ include "integrations.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "integrations.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "integrations.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/integrations/templates/deployment.yaml new file mode 100644 index 000000000..942f8c0a5 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/deployment.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "integrations.fullname" . }} + labels: + {{- include "integrations.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "integrations.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "integrations.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "integrations.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + - name: POSTGRES_STRING + value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:{{ .Values.global.postgresql.postgresqlPassword }}@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/integrations/templates/hpa.yaml new file mode 100644 index 000000000..cb9cf17db --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "integrations.fullname" . }} + labels: + {{- include "integrations.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "integrations.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/integrations/templates/ingress.yaml new file mode 100644 index 000000000..236409677 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "integrations.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "integrations.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/service.yaml b/scripts/helm/helm/openreplay/charts/integrations/templates/service.yaml new file mode 100644 index 000000000..77f0bb3c7 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "integrations.fullname" . }} + labels: + {{- include "integrations.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "integrations.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/integrations/templates/serviceaccount.yaml new file mode 100644 index 000000000..0bb50f48d --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "integrations.serviceAccountName" . }} + labels: + {{- include "integrations.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/integrations/templates/tests/test-connection.yaml new file mode 100644 index 000000000..ad5e040b5 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "integrations.fullname" . }}-test-connection" + labels: + {{- include "integrations.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "integrations.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/integrations/values.yaml b/scripts/helm/helm/openreplay/charts/integrations/values.yaml new file mode 100644 index 000000000..e35b79308 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/integrations/values.yaml @@ -0,0 +1,86 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/integrations + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "integrations" +fullnameOverride: "integrations" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + TOKEN_SECRET: secret_token_string # TODO: generate on buld + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/.helmignore b/scripts/helm/helm/openreplay/charts/nginx-ingress/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/Chart.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/Chart.yaml new file mode 100644 index 000000000..0019ec4f2 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: nginx-ingress +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/NOTES.txt new file mode 100644 index 000000000..8125afe9c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "nginx-ingress.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "nginx-ingress.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "nginx-ingress.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "nginx-ingress.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/_helpers.tpl new file mode 100644 index 000000000..b0f2808f1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "nginx-ingress.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nginx-ingress.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nginx-ingress.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "nginx-ingress.labels" -}} +helm.sh/chart: {{ include "nginx-ingress.chart" . }} +{{ include "nginx-ingress.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "nginx-ingress.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nginx-ingress.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "nginx-ingress.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "nginx-ingress.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/configMap.yaml new file mode 100644 index 000000000..f635b270c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/configMap.yaml @@ -0,0 +1,154 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx + namespace: {{ .Release.Namespace }} +data: + location.list: |- + location ~* /general_stats { + deny all; + } + location /healthz { + return 200 'OK'; + } + location ~ ^/(mobs|sessions-assets|frontend|static|sourcemaps|ios-images)/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio.db.svc.cluster.local:9000; + } + + location /minio/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_pass http://minio.db.svc.cluster.local:9000; + } + location /ingest/ { + rewrite ^/ingest/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header X-Forwarded-For $real_ip; + proxy_set_header X-Forwarded-Host $real_ip; + proxy_set_header X-Real-IP $real_ip; + proxy_set_header Host $host; + proxy_pass http://http-openreplay.app.svc.cluster.local; + proxy_read_timeout 300; + proxy_connect_timeout 120; + proxy_send_timeout 300; + } + location /grafana { + set $target http://monitoring-grafana.monitoring.svc.cluster.local; + rewrite ^/grafana/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_pass $target; + } + location /api/ { + rewrite ^/api/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; + } + location /assist/ { + rewrite ^/assist/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_pass http://utilities-openreplay.app.svc.cluster.local:9000; + } + location /assets/ { + rewrite ^/assets/(.*) /sessions-assets/$1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_pass http://minio.db.svc.cluster.local:9000; + } + location / { + index /index.html; + rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /frontend/index.html break; + include /etc/nginx/conf.d/compression.conf; + proxy_set_header Host $http_host; + proxy_pass http://minio.db.svc.cluster.local:9000/frontend/; + proxy_intercept_errors on; # see http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors + error_page 404 =200 /index.html; + } + compression.conf: |- + # Compression + gzip on; + gzip_comp_level 5; + gzip_min_length 256; # 256Bytes + gzip_proxied any; + gzip_vary on; + # Content types for compression + gzip_types + application/atom+xml + application/javascript + application/json + application/ld+json + application/manifest+json + application/rss+xml + application/vnd.geo+json + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/bmp + image/svg+xml + image/x-icon + text/cache-manifest + text/css + text/plain + ; + + sites.conf: |- + # Ref: https://github.com/openresty/openresty/#resolvconf-parsing + resolver local=on; + # Need real ip address for flags in replay. + # Some LBs will forward real ips as x-forwarded-for + # So making that as priority + map $http_x_forwarded_for $real_ip { + ~^(\d+\.\d+\.\d+\.\d+) $1; + default $remote_addr; + } + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + server { + listen 80 default_server; + listen [::]:80 default_server; + # server_name _; + {{ .Values.customServerConfigs }} + include /etc/nginx/conf.d/location.list; + client_max_body_size 10M; + } + server { + listen 443 ssl; + ssl_certificate /etc/secrets/site.crt; + ssl_certificate_key /etc/secrets/site.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + include /etc/nginx/conf.d/location.list; + client_max_body_size 10M; + } + diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/deployment.yaml new file mode 100644 index 000000000..7a64f5886 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/deployment.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nginx-ingress.fullname" . }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "nginx-ingress.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + nginxRolloutID: {{ randAlphaNum 5 | quote }} # Restart nginx after every deployment + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "nginx-ingress.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: https + containerPort: 443 + protocol: TCP + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + readinessProbe: + httpGet: + path: /healthz + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: nginx + mountPath: /etc/nginx/conf.d/ + - name: ssl + mountPath: /etc/secrets/ + volumes: + - name: nginx + configMap: + name: nginx + - name: ssl + secret: + secretName: ssl + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/hpa.yaml new file mode 100644 index 000000000..348f8f95b --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "nginx-ingress.fullname" . }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nginx-ingress.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/ingress.yaml new file mode 100644 index 000000000..63cfce077 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "nginx-ingress.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/secrets.yaml new file mode 100644 index 000000000..e5e4d7dd9 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/secrets.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: ssl +data: + ca.crt: "" + site.crt: "{{ .Values.ssl_certificate | b64enc }}" + site.key: "{{ .Values.ssl_privatekey | b64enc }}" diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/service.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/service.yaml new file mode 100644 index 000000000..94c034375 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nginx-ingress.fullname" . }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + # Make sure to get client ip + externalTrafficPolicy: Local + ports: + {{- range .Values.service.ports }} + - port: {{ .port }} + targetPort: {{ .targetPort }} + protocol: TCP + name: {{ .targetPort }} + {{- end }} + selector: + {{- include "nginx-ingress.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml new file mode 100644 index 000000000..bc0091029 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nginx-ingress.serviceAccountName" . }} + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml new file mode 100644 index 000000000..074cec518 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "nginx-ingress.fullname" . }}-test-connection" + labels: + {{- include "nginx-ingress.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "nginx-ingress.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/values.yaml b/scripts/helm/helm/openreplay/charts/nginx-ingress/values.yaml new file mode 100644 index 000000000..8c595aafa --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/nginx-ingress/values.yaml @@ -0,0 +1,173 @@ +# Default values for nginx-ingress. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: LoadBalancer + ports: + - port: 80 + targetPort: http + - port: 443 + targetPort: https + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + + +ssl_certificate: |- + -----BEGIN CERTIFICATE----- + MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL + BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 + NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh + bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ + LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w + hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds + QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX + sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o + immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l + fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ + ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ + OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u + tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p + CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA + YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 + z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs + MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z + qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA + dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p + 0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 + 5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh + 0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s + /JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP + XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc + HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG + T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi + qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S + w7yMPvE+TIh6+HwDZc2jNLkv/8tY + -----END CERTIFICATE----- + +ssl_privatekey: |- + -----BEGIN PRIVATE KEY----- + MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t + K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE + BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB + OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew + 6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK + aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 + hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK + lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 + VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 + RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK + Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh + QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe + K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s + dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 + CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh + ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl + 5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw + TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk + x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c + Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g + DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj + OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm + 5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p + /CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA + 0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 + tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E + ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy + HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 + IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA + PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q + p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO + GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw + 1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ + cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d + vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ + n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ + Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x + OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 + Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP + kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB + De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr + TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T + e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP + 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 + WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa + 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb + esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn + PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 + PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 + 7qINKoGovqXB1oAdopIl1z64e7MWVE4= + -----END PRIVATE KEY----- diff --git a/scripts/helm/helm/openreplay/charts/sink/.helmignore b/scripts/helm/helm/openreplay/charts/sink/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/sink/Chart.yaml b/scripts/helm/helm/openreplay/charts/sink/Chart.yaml new file mode 100644 index 000000000..ac86f8910 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: sink +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/sink/templates/NOTES.txt new file mode 100644 index 000000000..e49e60d4c --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "sink.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "sink.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "sink.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "sink.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/sink/templates/_helpers.tpl new file mode 100644 index 000000000..39f92ccc1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "sink.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sink.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "sink.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "sink.labels" -}} +helm.sh/chart: {{ include "sink.chart" . }} +{{ include "sink.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "sink.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sink.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "sink.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "sink.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/sink/templates/deployment.yaml new file mode 100644 index 000000000..ced1ee0c5 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/deployment.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "sink.fullname" . }} + labels: + {{- include "sink.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "sink.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "sink.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "sink.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: datadir + mountPath: /mnt/efs + {{- if eq .Values.pvc.name "hostPath" }} + volumes: + - name: datadir + hostPath: + # Ensure the file directory is created. + path: {{ .Values.pvc.hostMountPath }} + type: DirectoryOrCreate + {{- else }} + volumes: + - name: datadir + persistentVolumeClaim: + claimName: {{ .Values.pvc.name }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/sink/templates/hpa.yaml new file mode 100644 index 000000000..8f9a98f79 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "sink.fullname" . }} + labels: + {{- include "sink.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "sink.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/sink/templates/ingress.yaml new file mode 100644 index 000000000..ac5b25ba2 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "sink.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "sink.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/service.yaml b/scripts/helm/helm/openreplay/charts/sink/templates/service.yaml new file mode 100644 index 000000000..d2c0870c3 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "sink.fullname" . }} + labels: + {{- include "sink.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "sink.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/sink/templates/serviceaccount.yaml new file mode 100644 index 000000000..34986e78d --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "sink.serviceAccountName" . }} + labels: + {{- include "sink.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/sink/templates/tests/test-connection.yaml new file mode 100644 index 000000000..248381268 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "sink.fullname" . }}-test-connection" + labels: + {{- include "sink.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "sink.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/sink/values.yaml b/scripts/helm/helm/openreplay/charts/sink/values.yaml new file mode 100644 index 000000000..9d55ee370 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/sink/values.yaml @@ -0,0 +1,92 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/sink + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "sink" +fullnameOverride: "sink" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: {} + +pvc: + # This can be either persistentVolumeClaim or hostPath. + # In case of pvc, you'll have to provide the pvc name. + # For example + # name: openreplay-efs + name: hostPath + hostMountPath: /openreplay/storage/nfs + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/storage/.helmignore b/scripts/helm/helm/openreplay/charts/storage/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/storage/Chart.yaml b/scripts/helm/helm/openreplay/charts/storage/Chart.yaml new file mode 100644 index 000000000..3d122333f --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: storage +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/storage/templates/NOTES.txt new file mode 100644 index 000000000..217426ab3 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "storage.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "storage.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "storage.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "storage.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/storage/templates/_helpers.tpl new file mode 100644 index 000000000..9f87a2965 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "storage.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "storage.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "storage.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "storage.labels" -}} +helm.sh/chart: {{ include "storage.chart" . }} +{{ include "storage.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "storage.selectorLabels" -}} +app.kubernetes.io/name: {{ include "storage.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "storage.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "storage.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/storage/templates/deployment.yaml new file mode 100644 index 000000000..4ca3be6e2 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/deployment.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "storage.fullname" . }} + labels: + {{- include "storage.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "storage.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "storage.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "storage.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AWS_ACCESS_KEY_ID + value: '{{ .Values.global.s3.accessKey }}' + - name: AWS_SECRET_ACCESS_KEY + value: '{{ .Values.global.s3.secretKey }}' + - name: AWS_ENDPOINT + value: '{{ .Values.global.s3.endpoint }}' + - name: AWS_REGION_WEB + value: '{{ .Values.global.s3.region }}' + - name: AWS_REGION_IOS + value: '{{ .Values.global.s3.region }}' + - name: S3_BUCKET_WEB + value: '{{ .Values.global.s3.recordings_bucket }}' + - name: S3_BUCKET_IOS + value: '{{ .Values.global.s3.recordings_bucket }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' + - name: REDIS_STRING + value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' + - name: KAFKA_SERVERS + value: '{{ .Values.global.kafka.kafkaHost }}:{{ .Values.global.kafka.kafkaPort }}' + - name: KAFKA_USE_SSL + value: '{{ .Values.global.kafka.kafkaUseSsl }}' + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: datadir + mountPath: /mnt/efs + {{- if eq .Values.pvc.name "hostPath" }} + volumes: + - name: datadir + hostPath: + # Ensure the file directory is created. + path: {{ .Values.pvc.hostMountPath }} + type: DirectoryOrCreate + {{- else }} + volumes: + - name: datadir + persistentVolumeClaim: + claimName: {{ .Values.pvc.name }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/storage/templates/hpa.yaml new file mode 100644 index 000000000..d015e4bb2 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "storage.fullname" . }} + labels: + {{- include "storage.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "storage.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/storage/templates/ingress.yaml new file mode 100644 index 000000000..10bf66e98 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "storage.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "storage.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/service.yaml b/scripts/helm/helm/openreplay/charts/storage/templates/service.yaml new file mode 100644 index 000000000..f55083c92 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "storage.fullname" . }} + labels: + {{- include "storage.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "storage.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/storage/templates/serviceaccount.yaml new file mode 100644 index 000000000..a361acc50 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "storage.serviceAccountName" . }} + labels: + {{- include "storage.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/storage/templates/tests/test-connection.yaml new file mode 100644 index 000000000..84b0a6786 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "storage.fullname" . }}-test-connection" + labels: + {{- include "storage.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "storage.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/storage/values.yaml b/scripts/helm/helm/openreplay/charts/storage/values.yaml new file mode 100644 index 000000000..9f1dbfd91 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/storage/values.yaml @@ -0,0 +1,93 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/storage + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "storage" +fullnameOverride: "storage" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + FS_CLEAN_HRS: 24 + +pvc: + # This can be either persistentVolumeClaim or hostPath. + # In case of pvc, you'll have to provide the pvc name. + # For example + # name: openreplay-efs + name: hostPath + hostMountPath: /openreplay/storage/nfs + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/charts/utilities/.helmignore b/scripts/helm/helm/openreplay/charts/utilities/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/scripts/helm/helm/openreplay/charts/utilities/Chart.yaml b/scripts/helm/helm/openreplay/charts/utilities/Chart.yaml new file mode 100644 index 000000000..062a1ea73 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: utilities +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v1.3.6" diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/NOTES.txt b/scripts/helm/helm/openreplay/charts/utilities/templates/NOTES.txt new file mode 100644 index 000000000..323bc20c1 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "utilities.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "utilities.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "utilities.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "utilities.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/_helpers.tpl b/scripts/helm/helm/openreplay/charts/utilities/templates/_helpers.tpl new file mode 100644 index 000000000..8999db4be --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "utilities.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "utilities.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "utilities.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "utilities.labels" -}} +helm.sh/chart: {{ include "utilities.chart" . }} +{{ include "utilities.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "utilities.selectorLabels" -}} +app.kubernetes.io/name: {{ include "utilities.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "utilities.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "utilities.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/deployment.yaml b/scripts/helm/helm/openreplay/charts/utilities/templates/deployment.yaml new file mode 100644 index 000000000..628cda58d --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "utilities.fullname" . }} + labels: + {{- include "utilities.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "utilities.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "utilities.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "utilities.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- if .Values.global.enterpriseEditionLicense }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}-ee" + {{- else }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AWS_DEFAULT_REGION + value: "{{ .Values.global.s3.region }}" + - name: S3_HOST + {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} + value: 'https://{{ .Values.global.domainName }}' + {{- else}} + value: '{{ .Values.global.s3.endpoint }}' + {{- end}} + - name: S3_KEY + value: "{{ .Values.global.s3.accessKey }}" + - name: S3_SECRET + value: "{{ .Values.global.s3.secretKey }}" + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: '{{ $val }}' + {{- end}} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/hpa.yaml b/scripts/helm/helm/openreplay/charts/utilities/templates/hpa.yaml new file mode 100644 index 000000000..8944056ea --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "utilities.fullname" . }} + labels: + {{- include "utilities.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "utilities.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/ingress.yaml b/scripts/helm/helm/openreplay/charts/utilities/templates/ingress.yaml new file mode 100644 index 000000000..567cac846 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "utilities.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "utilities.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/service.yaml b/scripts/helm/helm/openreplay/charts/utilities/templates/service.yaml new file mode 100644 index 000000000..c9afad7d5 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "utilities.fullname" . }} + labels: + {{- include "utilities.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "utilities.selectorLabels" . | nindent 4 }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/serviceaccount.yaml b/scripts/helm/helm/openreplay/charts/utilities/templates/serviceaccount.yaml new file mode 100644 index 000000000..dd5c35012 --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "utilities.serviceAccountName" . }} + labels: + {{- include "utilities.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/tests/test-connection.yaml b/scripts/helm/helm/openreplay/charts/utilities/templates/tests/test-connection.yaml new file mode 100644 index 000000000..44b72f68d --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "utilities.fullname" . }}-test-connection" + labels: + {{- include "utilities.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "utilities.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/scripts/helm/helm/openreplay/charts/utilities/values.yaml b/scripts/helm/helm/openreplay/charts/utilities/values.yaml new file mode 100644 index 000000000..6c3e0056f --- /dev/null +++ b/scripts/helm/helm/openreplay/charts/utilities/values.yaml @@ -0,0 +1,85 @@ +# Default values for openreplay. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rg.fr-par.scw.cloud/foss/utilities + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "utililities" +fullnameOverride: "utililities" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 9000 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 5 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: {} + + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/scripts/helm/helm/openreplay/files/clickhouse.sh b/scripts/helm/helm/openreplay/files/clickhouse.sh new file mode 100644 index 000000000..80cb98fa3 --- /dev/null +++ b/scripts/helm/helm/openreplay/files/clickhouse.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +clickhousedir=/opt/openreplay/openreplay/scripts/helm/db/init_dbs/clickhouse + +function migrate() { + echo "Starting clickhouse migration" + migration_versions=$1 + for version in $migration_versions; do + echo "Migrating clickhouse version $version" + # For now, we can ignore the clickhouse db inject errors. + # TODO: Better error handling in script + clickhouse-client -h clickhouse.db.svc.cluster.local --port 9000 < ${clickhousedir}/${version}/${version}.sql || true + done +} + +function init() { + echo "Initializing clickhouse" + for file in `ls ${clickhousedir}/create/*.sql`; do + echo "Injecting $file" + clickhouse-client -h clickhouse.db.svc.cluster.local --port 9000 < $file + done +} + +# /bin/bash clickhouse.sh migrate $migration_versions +case "$1" in + migrate) + migrate $2 + ;; + init) + init + ;; + *) + echo "Unknown operation for clickhouse migration; exiting." + exit 1 + ;; +esac diff --git a/scripts/helm/helm/openreplay/files/dbops.sh b/scripts/helm/helm/openreplay/files/dbops.sh new file mode 100644 index 000000000..5df6eb727 --- /dev/null +++ b/scripts/helm/helm/openreplay/files/dbops.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +cd $(dirname $0) + +function migration() { + ls -la /opt/openreplay/openreplay + db=$1 + + # Checking if previous app version is set. + if [[ $PREVIOUS_APP_VERSION == "" ]]; then + echo "Previous app verision to be migrated is not set. Rerun using --set fromVersion=v1.3.5" + exit 100 + fi + + if [[ $PREVIOUS_APP_VERSION == $CHART_APP_VERSION ]]; then + echo "No application version change. Not upgrading." + exit 0 + fi + + # Checking migration versions + cd /opt/openreplay/openreplay/scripts/helm + migration_versions=(`ls -l db/init_dbs/$db | grep -E ^d | awk -v number=${PREVIOUS_APP_VERSION} '$NF > number {print $NF}' | grep -v create`) + echo "Migration version: $migration_versions" + + cd - + + case "$1" in + postgresql) + /bin/bash postgresql.sh migrate $migration_versions + ;; + minio) + /bin/bash minio.sh migrate $migration_versions + ;; + clickhouse) + /bin/bash clickhouse.sh migrate $migration_versions + ;; + kafka) + /bin/bash kafka.sh migrate $migration_versions + ;; + *) + echo "Unknown operation for db migration; exiting." + exit 1 + ;; + esac +} + +function init(){ + case $1 in + postgresql) + /bin/bash postgresql.sh init + ;; + minio) + /bin/bash minio.sh migrate $migration_versions + ;; + clickhouse) + /bin/bash clickhouse.sh init + ;; + kafka) + /bin/bash kafka.sh init + ;; + *) + echo "Unknown operation for db init; exiting." + exit 1 + ;; + + esac +} + + +# dbops.sh true(upgrade) clickhouse +case "$1" in + "false") + init $2 + ;; + "true") + migration $2 + ;; + *) + echo "Unknown operation for db migration; exiting." + exit 1 + ;; +esac diff --git a/scripts/helm/helm/openreplay/files/kafka.sh b/scripts/helm/helm/openreplay/files/kafka.sh new file mode 100644 index 000000000..9105f3b00 --- /dev/null +++ b/scripts/helm/helm/openreplay/files/kafka.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +topics=( + "raw" + "raw-ios" + "trigger" + "cache" + "analytics" + ) + +touch /tmp/config.txt + +if [[ $KAFKA_SSL == "true" ]]; then + echo 'securty.protocol=SSL' > /tmp/config.txt +fi + +function init() { + echo "Initializing kafka" + for topic in ${topics[*]}; do + echo "Creating topic: $topic" + # TODO: Have to check an idempotent way of creating topics. + kafka-topics.sh --create --bootstrap-server ${KAFKA_HOST}:${KAFKA_PORT} --replication-factor 2 --partitions 16 --topic ${topic} --command-config /tmp/config.txt || true + kafka-configs.sh --bootstrap-server ${KAFKA_HOST}:${KAFKA_PORT} --entity-type topics --alter --add-config retention.ms=3456000000 --entity-name=${topic} --command-config /tmp/config.txt || true + done +} + +# /bin/bash kafka.sh migrate $migration_versions +case "$1" in + migrate) + init + ;; + init) + init + ;; + *) + echo "Unknown operation for kafka migration; exiting." + exit 1 + ;; +esac diff --git a/scripts/helm/helm/openreplay/files/minio.sh b/scripts/helm/helm/openreplay/files/minio.sh new file mode 100644 index 000000000..2bb7682f3 --- /dev/null +++ b/scripts/helm/helm/openreplay/files/minio.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + + +cd /tmp + +buckets=("mobs" "sessions-assets" "static" "sourcemaps") + +mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY + +function init() { +echo "Initializing minio" + +for bucket in ${buckets[*]}; do +mc mb minio/${bucket} || true +mc ilm import minio/${bucket} < Date: Sat, 15 Jan 2022 01:16:14 +0530 Subject: [PATCH 097/206] chore(nginx): Adding x-forwarded-proto map x-forwarded-proto is not standard aginst all LB, If nginx is offloading SSL, then there won't be any x-forwarded-proto. So, default we're taking $http_x_forwarded_proto else $scheme. Signed-off-by: Rajesh Rajendran --- .../nginx-ingress/nginx-ingress/templates/configmap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml b/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml index bf41a28c2..ea59aa82d 100644 --- a/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml +++ b/scripts/helm/nginx-ingress/nginx-ingress/templates/configmap.yaml @@ -62,7 +62,7 @@ data: proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $origin_proto; proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; } location /assist/ { @@ -133,6 +133,10 @@ data: default upgrade; '' close; } + map $http_x_forwarded_proto $origin_proto { + default $http_x_forwarded_proto; + '' $scheme; + } server { listen 80 default_server; listen [::]:80 default_server; From 9f34cd1614b369fc023a77fb37fa4361bb4a9b0f Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 15 Jan 2022 02:08:52 +0530 Subject: [PATCH 098/206] docs(vars): update documentation. Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/vars.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/helm/helm/vars.yaml b/scripts/helm/helm/vars.yaml index a8e63440d..fac420bbc 100644 --- a/scripts/helm/helm/vars.yaml +++ b/scripts/helm/helm/vars.yaml @@ -62,7 +62,32 @@ global: # limits: # cpu: 1 -# If you want to redirect nginx http to https # nginx-ingress: +# +### If you want to redirect nginx http to https # customServerConfigs: | # return 301 https://$host$request_uri; +# +### Change the ssl certificates +# +### Public certificate ( content from site.crt, mind the indentation ) +# ssl_certificate: |- +# -----BEGIN CERTIFICATE----- +# MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL +# BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 +# NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh +# -----END CERTIFICATE----- +# +### Private certificate ( content from site.key, mind the indentation. ) +# ssl_privatekey: |- +# -----BEGIN PRIVATE KEY----- +# TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T +# e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP +# 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 +# WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa +# 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb +# esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn +# PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 +# PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 +# 7qINKoGovqXB1oAdopIl1z64e7MWVE4= +# -----END PRIVATE KEY----- From 855b9528f1e11c9ec7854c9d3e19844b44fe97ec Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 15 Jan 2022 02:19:22 +0530 Subject: [PATCH 099/206] chore(vars): updating version Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/vars.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helm/helm/vars.yaml b/scripts/helm/helm/vars.yaml index fac420bbc..2c279c264 100644 --- a/scripts/helm/helm/vars.yaml +++ b/scripts/helm/helm/vars.yaml @@ -1,3 +1,4 @@ +fromVersion: "v1.3.6" # Databases specific variables postgresql: &postgres # For genrating psswords From 8dc11b385047ee5b98026b84404641943a730a86 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 15 Jan 2022 02:27:41 +0530 Subject: [PATCH 100/206] chore(install): fixing typo Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/openreplay/files/dbops.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/helm/openreplay/files/dbops.sh b/scripts/helm/helm/openreplay/files/dbops.sh index 5df6eb727..542b6ffcf 100644 --- a/scripts/helm/helm/openreplay/files/dbops.sh +++ b/scripts/helm/helm/openreplay/files/dbops.sh @@ -8,7 +8,7 @@ function migration() { # Checking if previous app version is set. if [[ $PREVIOUS_APP_VERSION == "" ]]; then - echo "Previous app verision to be migrated is not set. Rerun using --set fromVersion=v1.3.5" + echo "Previous app version to be migrated is not set. Rerun using --set fromVersion=v1.3.5" exit 100 fi From 5d1d64364673ec3b06e656b4788cf5bfb186a8bc Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 15 Jan 2022 02:30:36 +0530 Subject: [PATCH 101/206] docs(vars): example for chalice envs Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/vars.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/helm/helm/vars.yaml b/scripts/helm/helm/vars.yaml index 2c279c264..73028ee8f 100644 --- a/scripts/helm/helm/vars.yaml +++ b/scripts/helm/helm/vars.yaml @@ -46,6 +46,12 @@ global: chalice: env: jwt_secret: "SetARandomStringHere" + # EMAIL_HOST: '' + # EMAIL_PORT: '587' + # EMAIL_USER: '' + # EMAIL_PASSWORD: '' + # EMAIL_USE_TLS: 'true' + # EMAIL_FROM: OpenReplay enterpriseEditionLicense: "" domainName: "" From 95c8af473f28d0e91487bcf4cb5fe2ee90dba463 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 15 Jan 2022 02:53:53 +0530 Subject: [PATCH 102/206] chore(vars): comments update Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/vars.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/helm/helm/vars.yaml b/scripts/helm/helm/vars.yaml index 73028ee8f..f15033aa3 100644 --- a/scripts/helm/helm/vars.yaml +++ b/scripts/helm/helm/vars.yaml @@ -9,7 +9,13 @@ postgresql: &postgres postgresqlUser: "postgres" postgresqlDatabase: "postgres" +clickhouse: {} + # For enterpriseEdition + # enabled: true + kafka: &kafka + # For enterpriseEdition + # enabled: true kafkaHost: "kafka.db.svc.cluster.local" kafkaPort: "9092" kafkaUseSsl: "false" @@ -21,7 +27,7 @@ redis: &redis minio: global: minio: - # For genrating psswords + # For generating passwords # `openssl rand -hex 20` accessKey: "changeMeMinioAccessKey" secretKey: "changeMeMinioPassword" From fd34f08490ca03ab73b52f4e3c6a6224c4ca753d Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 15 Jan 2022 02:57:41 +0530 Subject: [PATCH 103/206] chore(vars): not download vars if present Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/init.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/helm/helm/init.sh b/scripts/helm/helm/init.sh index 3e6d2a96b..74a113c4a 100644 --- a/scripts/helm/helm/init.sh +++ b/scripts/helm/helm/init.sh @@ -75,26 +75,27 @@ randomPass() { ## Don't override existing variables file. [[ -f vars.yaml ]] && { - warn "Existing Variables file. Not overriding." + warn "Existing Variables file. Not downloading." }|| { +info " Downloading vars file" +curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${version}/scripts/helm/helm/vars.yaml + +} + [[ -z $DOMAIN_NAME ]] && { fatal "DOMAIN_NAME variable is empty. Rerun the script" info "curl -sL get.openreplay.com | sudo DOMAIN_NAME=openerplay.mycomp.org bash - " exit 101 } -info " Downloading vars file" -curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${version}/scripts/helm/helm/vars.yaml - info "Creating dynamic passwords" sed -i "s/postgresqlPassword: \"changeMePassword\"/postgresqlPassword: \"$(randomPass)\"/g" vars.yaml sed -i "s/accessKey: \"changeMeMinioAccessKey\"/accessKey: \"$(randomPass)\"/g" vars.yaml sed -i "s/secretKey: \"changeMeMinioPassword\"/secretKey: \"$(randomPass)\"/g" vars.yaml sed -i "s/jwt_secret: \"SetARandomStringHere\"/jwt_secret: \"$(randomPass)\"/g" vars.yaml -sed -i "s/domainName: .*/domainName: \"${DOMAIN_NAME}\"/g" vars.yaml +sed -i "s/domainName: \"\"/domainName: \"${DOMAIN_NAME}\"/g" vars.yaml -} ## Installing OpenReplay info "Installing databases" From a7308948923d1d43371a71472860f35be8b5bb21 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sun, 16 Jan 2022 16:44:34 +0530 Subject: [PATCH 104/206] fix(install): fatal log Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/init.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/helm/helm/init.sh b/scripts/helm/helm/init.sh index 74a113c4a..bb327b347 100644 --- a/scripts/helm/helm/init.sh +++ b/scripts/helm/helm/init.sh @@ -84,9 +84,7 @@ curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${v } [[ -z $DOMAIN_NAME ]] && { -fatal "DOMAIN_NAME variable is empty. Rerun the script" -info "curl -sL get.openreplay.com | sudo DOMAIN_NAME=openerplay.mycomp.org bash - " -exit 101 +fatal 'DOMAIN_NAME variable is empty. Rerun the script `curl -sL get.openreplay.com | sudo DOMAIN_NAME=openerplay.mycomp.org bash - `' } info "Creating dynamic passwords" From 94b7ed113542d9287ed3a4918f33bfb2598d61f8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sun, 16 Jan 2022 17:14:42 +0100 Subject: [PATCH 105/206] feat(api): EE SSO handle deleted-user --- ee/api/chalicelib/core/users.py | 54 +++++++++++++++++++++++++++++++++ ee/api/routers/saml.py | 19 ++++++++---- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/ee/api/chalicelib/core/users.py b/ee/api/chalicelib/core/users.py index 8f1ef7955..ce5bcca5d 100644 --- a/ee/api/chalicelib/core/users.py +++ b/ee/api/chalicelib/core/users.py @@ -736,3 +736,57 @@ def create_sso_user(tenant_id, email, admin, name, origin, role_id, internal_id= query ) return helper.dict_to_camel_case(cur.fetchone()) + + +def restore_sso_user(user_id, tenant_id, email, admin, name, origin, role_id, internal_id=None): + with pg_client.PostgresClient() as cur: + query = cur.mogrify(f"""\ + WITH u AS ( + UPDATE public.users + SET tenant_id= %(tenantId)s, + role= %(role)s, + name= %(name)s, + data= %(data)s, + origin= %(origin)s, + internal_id= %(internal_id)s, + role_id= %(role_id)s, + deleted_at= NULL, + created_at= default, + api_key= default, + jwt_iat= NULL, + appearance= default, + weekly_report= default + WHERE user_id = %(user_id)s + RETURNING * + ), + au AS ( + UPDATE public.basic_authentication + SET password= default, + generated_password= default, + invitation_token= default, + invited_at= default, + change_pwd_token= default, + change_pwd_expire_at= default, + changed_at= NULL + WHERE user_id = %(user_id)s + RETURNING user_id + ) + SELECT u.user_id AS id, + u.email, + u.role, + u.name, + TRUE AS change_password, + (CASE WHEN u.role = 'owner' THEN TRUE ELSE FALSE END) AS super_admin, + (CASE WHEN u.role = 'admin' THEN TRUE ELSE FALSE END) AS admin, + (CASE WHEN u.role = 'member' THEN TRUE ELSE FALSE END) AS member, + u.appearance, + origin + FROM u;""", + {"tenantId": tenant_id, "email": email, "internal_id": internal_id, + "role": "admin" if admin else "member", "name": name, "origin": origin, + "role_id": role_id, "data": json.dumps({"lastAnnouncementView": TimeUTC.now()}), + "user_id": user_id}) + cur.execute( + query + ) + return helper.dict_to_camel_case(cur.fetchone()) diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index c9f074324..50723a1db 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -24,7 +24,6 @@ async def start_sso(request: Request): return RedirectResponse(url=sso_built_url) -# @public_app.post('/sso/saml2/acs', tags=["saml2"], content_types=['application/x-www-form-urlencoded']) @public_app.post('/sso/saml2/acs', tags=["saml2"]) async def process_sso_assertion(request: Request): req = await prepare_request(request=request) @@ -77,11 +76,19 @@ async def process_sso_assertion(request: Request): or admin_privileges[0].lower() == "false") if existing is None: - print("== new user ==") - users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=admin_privileges, - origin=SAML2_helper.get_saml2_provider(), - name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), - internal_id=internal_id, role_id=role["roleId"]) + deleted = users.get_deleted_user_by_email(auth.get_nameid()) + if deleted is not None: + print("== restore deleted user ==") + users.restore_sso_user(user_id=deleted["userId"], tenant_id=t['tenantId'], email=email, + admin=admin_privileges, origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + print("== new user ==") + users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=admin_privileges, + origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) else: if t['tenantId'] != existing["tenantId"]: print("user exists for a different tenant") From c642da11d3379149c195236bb82b1df07b6bbc24 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 00:56:57 +0530 Subject: [PATCH 106/206] chore(install): prepending commits with sudo Signed-off-by: Rajesh Rajendran --- scripts/helm/helm/init.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/helm/helm/init.sh b/scripts/helm/helm/init.sh index bb327b347..8b926cf50 100644 --- a/scripts/helm/helm/init.sh +++ b/scripts/helm/helm/init.sh @@ -15,19 +15,16 @@ fatal() exit 1 } -version=${OPENREPLAY_VERSION:-'v1.3.6'} - -# Check whether script is root user -[[ $(id -u) -eq 0 ]] || { - fatal "Please run the script as root user" - exit 100 -} +version=${OPENREPLAY_VERSION:-'v1.4.0'} +usr=`whoami` # Installing k3s curl -sL https://get.k3s.io | sudo K3S_KUBECONFIG_MODE="644" INSTALL_K3S_VERSION='v1.19.5+k3s2' INSTALL_K3S_EXEC="--no-deploy=traefik" sh - mkdir ~/.kube -cp /etc/rancher/k3s/k3s.yaml ~/.kube/config +sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config chmod 0644 ~/.kube/config +sudo chown -R $usr ~/.kube/config + ## installing kubectl which kubectl &> /dev/null || { @@ -64,8 +61,8 @@ if [[ $? -ne 0 ]]; then fi ## Installing openssl -apt update &> /dev/null -apt install openssl -y &> /dev/null +sudo apt update &> /dev/null +sudo apt install openssl -y &> /dev/null randomPass() { openssl rand -hex 10 @@ -84,7 +81,7 @@ curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${v } [[ -z $DOMAIN_NAME ]] && { -fatal 'DOMAIN_NAME variable is empty. Rerun the script `curl -sL get.openreplay.com | sudo DOMAIN_NAME=openerplay.mycomp.org bash - `' +fatal 'DOMAIN_NAME variable is empty. Rerun the script `DOMAIN_NAME=openreplay.mycomp.org bash init.sh `' } info "Creating dynamic passwords" @@ -100,4 +97,3 @@ info "Installing databases" helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./vars.yaml --atomic info "Installing application" helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./vars.yaml --atomic - From f8c1c6f2862afdd8f956d8c85c6deb8146fa0019 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 00:59:16 +0530 Subject: [PATCH 107/206] chore(openreplaycli): Adding installation via helm Signed-off-by: Rajesh Rajendran --- scripts/helm/openreplay-cli | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/helm/openreplay-cli b/scripts/helm/openreplay-cli index 93478e2d6..6b6fae514 100755 --- a/scripts/helm/openreplay-cli +++ b/scripts/helm/openreplay-cli @@ -51,7 +51,8 @@ EOF [ -d | --status ] [ -v | --verbose ] [ -l | --logs SERVICE ] - [ -i | --install SERVICE ] + [ -i | --legacy-install SERVICE ] + [ -I | --helm-install SERVICE ] [ -s | --stop SERVICE|all ] [ -S | --start SERVICE|all ] [ -r | --restart SERVICE|all ]" @@ -103,10 +104,14 @@ restart() { kubectl rollout restart -n app deployment $1-openreplay } -install() { +legacyInstall() { bash kube-install.sh --app $1 } +helmInstall() { + helm upgrade --install openreplay helmv2/openreplay -f helmv2/vars.yaml +} + upgrade() { sed -i "s/tag:.*/ tag: 'latest'/g" ./app/$1.yaml } @@ -122,7 +127,7 @@ status() { [[ $# -eq 0 ]] && usage && exit 1 -PARSED_ARGUMENTS=$(color getopt -a -n openreplay-cli -o vhds:S:l:r:i: --long verbose,help,status,start:,stop:,logs:,restart:,install: -- "$@") +PARSED_ARGUMENTS=$(color getopt -a -n openreplay-cli -o vhds:S:l:r:i:I: --long verbose,help,status,start:,stop:,logs:,restart:,legacy-install:,helm-install: -- "$@") VALID_ARGUMENTS=$? if [[ "$VALID_ARGUMENTS" != "0" ]]; then usage @@ -139,7 +144,8 @@ do -S | --start) start $2 ; shift 2 ;; -l | --logs) logs "$2" ; shift 2 ;; -r | --restart) restart "$2" ; shift 2 ;; - -i | --install) install "$2" ; shift 2 ;; + -i | --legacy-install) legacyInstall "$2" ; shift 2 ;; + -I | --helm-install) helmInstall "$2" ; shift 2 ;; # -- means the end of the arguments; drop this, and break out of the while loop --) shift; break ;; # If invalid options were passed, then getopt should have reported an error, From 4d14e177120a920dae880c3fad6e959c1ccd4642 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 01:15:08 +0530 Subject: [PATCH 108/206] chore(helm): proper folder name Signed-off-by: Rajesh Rajendran --- scripts/helm/{helm => helmcharts}/README.md | 0 .../helm/{helm => helmcharts}/databases/.helmignore | 0 .../helm/{helm => helmcharts}/databases/Chart.yaml | 0 .../databases/charts/clickhouse/.helmignore | 0 .../databases/charts/clickhouse/Chart.yaml | 0 .../charts/clickhouse/templates/_helpers.tpl | 0 .../charts/clickhouse/templates/service.yaml | 0 .../charts/clickhouse/templates/serviceaccount.yaml | 0 .../charts/clickhouse/templates/statefulset.yaml | 0 .../databases/charts/clickhouse/values.yaml | 0 .../databases/charts/kafka/.helmignore | 0 .../databases/charts/kafka/Chart.yaml | 0 .../databases/charts/kafka/README.md | 0 .../charts/kafka/charts/zookeeper/.helmignore | 0 .../charts/kafka/charts/zookeeper/Chart.yaml | 0 .../charts/kafka/charts/zookeeper/README.md | 0 .../kafka/charts/zookeeper/templates/NOTES.txt | 0 .../kafka/charts/zookeeper/templates/_helpers.tpl | 0 .../kafka/charts/zookeeper/templates/configmap.yaml | 0 .../charts/zookeeper/templates/metrics-svc.yaml | 0 .../charts/zookeeper/templates/networkpolicy.yaml | 0 .../zookeeper/templates/poddisruptionbudget.yaml | 0 .../charts/zookeeper/templates/prometheusrules.yaml | 0 .../kafka/charts/zookeeper/templates/secrets.yaml | 0 .../charts/zookeeper/templates/serviceaccount.yaml | 0 .../charts/zookeeper/templates/servicemonitor.yaml | 0 .../charts/zookeeper/templates/statefulset.yaml | 0 .../charts/zookeeper/templates/svc-headless.yaml | 0 .../kafka/charts/zookeeper/templates/svc.yaml | 0 .../kafka/charts/zookeeper/values-production.yaml | 0 .../charts/kafka/charts/zookeeper/values.yaml | 0 .../databases/charts/kafka/files/jks/README.md | 0 .../databases/charts/kafka/kafka.yaml | 0 .../databases/charts/kafka/requirements.lock | 0 .../databases/charts/kafka/requirements.yaml | 0 .../databases/charts/kafka/templates/NOTES.txt | 0 .../databases/charts/kafka/templates/_helpers.tpl | 0 .../databases/charts/kafka/templates/configmap.yaml | 0 .../charts/kafka/templates/extra-list.yaml | 0 .../charts/kafka/templates/jaas-secret.yaml | 0 .../charts/kafka/templates/jks-secret.yaml | 0 .../charts/kafka/templates/jmx-configmap.yaml | 0 .../charts/kafka/templates/jmx-metrics-svc.yaml | 0 .../kafka/templates/kafka-metrics-deployment.yaml | 0 .../charts/kafka/templates/kafka-metrics-svc.yaml | 0 .../charts/kafka/templates/log4j-configmap.yaml | 0 .../charts/kafka/templates/poddisruptionbudget.yaml | 0 .../databases/charts/kafka/templates/role.yaml | 0 .../charts/kafka/templates/rolebinding.yaml | 0 .../charts/kafka/templates/scripts-configmap.yaml | 0 .../charts/kafka/templates/serviceaccount.yaml | 0 .../kafka/templates/servicemonitor-jmx-metrics.yaml | 0 .../kafka/templates/servicemonitor-metrics.yaml | 0 .../charts/kafka/templates/statefulset.yaml | 0 .../charts/kafka/templates/svc-external-access.yaml | 0 .../charts/kafka/templates/svc-headless.yaml | 0 .../databases/charts/kafka/templates/svc.yaml | 0 .../databases/charts/kafka/values-production.yaml | 0 .../databases/charts/kafka/values.yaml | 0 .../databases/charts/minio/.helmignore | 0 .../databases/charts/minio/Chart.yaml | 0 .../databases/charts/minio/README.md | 0 .../charts/minio/ci/values-production.yaml | 0 .../databases/charts/minio/templates/NOTES.txt | 0 .../databases/charts/minio/templates/_helpers.tpl | 0 .../minio/templates/deployment-standalone.yaml | 0 .../databases/charts/minio/templates/ingress.yaml | 0 .../charts/minio/templates/networkpolicy.yaml | 0 .../charts/minio/templates/pvc-standalone.yaml | 0 .../databases/charts/minio/templates/secrets.yaml | 0 .../databases/charts/minio/templates/service.yaml | 0 .../charts/minio/templates/serviceaccount.yaml | 0 .../charts/minio/templates/statefulset.yaml | 0 .../charts/minio/templates/svc-headless.yaml | 0 .../charts/minio/templates/tls-secrets.yaml | 0 .../databases/charts/minio/values-production.yaml | 0 .../databases/charts/minio/values.yaml | 0 .../databases/charts/postgresql/.helmignore | 0 .../databases/charts/postgresql/Chart.yaml | 0 .../databases/charts/postgresql/README.md | 0 .../charts/postgresql/charts/common/.helmignore | 0 .../charts/postgresql/charts/common/Chart.yaml | 0 .../charts/postgresql/charts/common/README.md | 0 .../charts/common/templates/_affinities.tpl | 0 .../charts/common/templates/_capabilities.tpl | 0 .../postgresql/charts/common/templates/_errors.tpl | 0 .../postgresql/charts/common/templates/_images.tpl | 0 .../postgresql/charts/common/templates/_labels.tpl | 0 .../postgresql/charts/common/templates/_names.tpl | 0 .../postgresql/charts/common/templates/_secrets.tpl | 0 .../postgresql/charts/common/templates/_storage.tpl | 0 .../charts/common/templates/_tplvalues.tpl | 0 .../postgresql/charts/common/templates/_utils.tpl | 0 .../charts/common/templates/_validations.tpl | 0 .../charts/common/templates/_warnings.tpl | 0 .../charts/postgresql/charts/common/values.yaml | 0 .../charts/postgresql/ci/commonAnnotations.yaml | 0 .../charts/postgresql/ci/default-values.yaml | 0 .../postgresql/ci/shmvolume-disabled-values.yaml | 0 .../databases/charts/postgresql/files/README.md | 0 .../charts/postgresql/files/conf.d/README.md | 0 .../files/docker-entrypoint-initdb.d/README.md | 0 .../databases/charts/postgresql/postgresql.yaml | 0 .../databases/charts/postgresql/requirements.lock | 0 .../databases/charts/postgresql/requirements.yaml | 0 .../databases/charts/postgresql/templates/NOTES.txt | 0 .../charts/postgresql/templates/_helpers.tpl | 0 .../charts/postgresql/templates/configmap.yaml | 0 .../templates/extended-config-configmap.yaml | 0 .../charts/postgresql/templates/extra-list.yaml | 0 .../templates/initialization-configmap.yaml | 0 .../postgresql/templates/metrics-configmap.yaml | 0 .../charts/postgresql/templates/metrics-svc.yaml | 0 .../charts/postgresql/templates/networkpolicy.yaml | 0 .../postgresql/templates/podsecuritypolicy.yaml | 0 .../charts/postgresql/templates/prometheusrule.yaml | 0 .../databases/charts/postgresql/templates/role.yaml | 0 .../charts/postgresql/templates/rolebinding.yaml | 0 .../charts/postgresql/templates/secrets.yaml | 0 .../charts/postgresql/templates/serviceaccount.yaml | 0 .../charts/postgresql/templates/servicemonitor.yaml | 0 .../postgresql/templates/statefulset-slaves.yaml | 0 .../charts/postgresql/templates/statefulset.yaml | 0 .../charts/postgresql/templates/svc-headless.yaml | 0 .../charts/postgresql/templates/svc-read.yaml | 0 .../databases/charts/postgresql/templates/svc.yaml | 0 .../charts/postgresql/values-production.yaml | 0 .../databases/charts/postgresql/values.schema.json | 0 .../databases/charts/postgresql/values.yaml | 0 .../databases/charts/redis/.helmignore | 0 .../databases/charts/redis/Chart.lock | 0 .../databases/charts/redis/Chart.yaml | 0 .../databases/charts/redis/README.md | 0 .../charts/redis/charts/common/.helmignore | 0 .../databases/charts/redis/charts/common/Chart.yaml | 0 .../databases/charts/redis/charts/common/README.md | 0 .../redis/charts/common/templates/_affinities.tpl | 0 .../redis/charts/common/templates/_capabilities.tpl | 0 .../redis/charts/common/templates/_errors.tpl | 0 .../redis/charts/common/templates/_images.tpl | 0 .../redis/charts/common/templates/_ingress.tpl | 0 .../redis/charts/common/templates/_labels.tpl | 0 .../charts/redis/charts/common/templates/_names.tpl | 0 .../redis/charts/common/templates/_secrets.tpl | 0 .../redis/charts/common/templates/_storage.tpl | 0 .../redis/charts/common/templates/_tplvalues.tpl | 0 .../charts/redis/charts/common/templates/_utils.tpl | 0 .../redis/charts/common/templates/_warnings.tpl | 0 .../common/templates/validations/_cassandra.tpl | 0 .../common/templates/validations/_mariadb.tpl | 0 .../common/templates/validations/_mongodb.tpl | 0 .../common/templates/validations/_postgresql.tpl | 0 .../charts/common/templates/validations/_redis.tpl | 0 .../common/templates/validations/_validations.tpl | 0 .../charts/redis/charts/common/values.yaml | 0 .../databases/charts/redis/ci/default-values.yaml | 0 .../charts/redis/ci/extra-flags-values.yaml | 0 .../charts/redis/ci/production-sentinel-values.yaml | 0 .../charts/redis/img/redis-cluster-topology.png | Bin .../databases/charts/redis/img/redis-topology.png | Bin .../databases/charts/redis/templates/NOTES.txt | 0 .../databases/charts/redis/templates/_helpers.tpl | 0 .../charts/redis/templates/configmap-scripts.yaml | 0 .../databases/charts/redis/templates/configmap.yaml | 0 .../charts/redis/templates/headless-svc.yaml | 0 .../charts/redis/templates/health-configmap.yaml | 0 .../charts/redis/templates/metrics-prometheus.yaml | 0 .../templates/metrics-sentinel-prometheus.yaml | 0 .../redis/templates/metrics-sentinel-svc.yaml | 0 .../charts/redis/templates/metrics-svc.yaml | 0 .../charts/redis/templates/networkpolicy.yaml | 0 .../databases/charts/redis/templates/pdb.yaml | 0 .../charts/redis/templates/prometheusrule.yaml | 0 .../databases/charts/redis/templates/psp.yaml | 0 .../redis/templates/redis-master-statefulset.yaml | 0 .../charts/redis/templates/redis-master-svc.yaml | 0 .../redis/templates/redis-node-statefulset.yaml | 0 .../charts/redis/templates/redis-role.yaml | 0 .../charts/redis/templates/redis-rolebinding.yaml | 0 .../redis/templates/redis-serviceaccount.yaml | 0 .../redis/templates/redis-slave-statefulset.yaml | 0 .../charts/redis/templates/redis-slave-svc.yaml | 0 .../redis/templates/redis-with-sentinel-svc.yaml | 0 .../databases/charts/redis/templates/secret.yaml | 0 .../databases/charts/redis/values.schema.json | 0 .../databases/charts/redis/values.yaml | 0 .../databases/templates/NOTES.txt | 0 .../databases/templates/_helpers.tpl | 0 .../helm/{helm => helmcharts}/databases/values.yaml | 0 scripts/helm/{helm => helmcharts}/init.sh | 0 .../{helm => helmcharts}/openreplay/.helmignore | 0 .../helm/{helm => helmcharts}/openreplay/Chart.yaml | 0 .../openreplay/charts/alerts/.helmignore | 0 .../openreplay/charts/alerts/Chart.yaml | 0 .../openreplay/charts/alerts/templates/NOTES.txt | 0 .../openreplay/charts/alerts/templates/_helpers.tpl | 0 .../charts/alerts/templates/deployment.yaml | 0 .../openreplay/charts/alerts/templates/hpa.yaml | 0 .../openreplay/charts/alerts/templates/ingress.yaml | 0 .../openreplay/charts/alerts/templates/service.yaml | 0 .../charts/alerts/templates/serviceaccount.yaml | 0 .../alerts/templates/tests/test-connection.yaml | 0 .../openreplay/charts/alerts/values.yaml | 0 .../openreplay/charts/assets/.helmignore | 0 .../openreplay/charts/assets/Chart.yaml | 0 .../openreplay/charts/assets/templates/NOTES.txt | 0 .../openreplay/charts/assets/templates/_helpers.tpl | 0 .../charts/assets/templates/deployment.yaml | 0 .../openreplay/charts/assets/templates/hpa.yaml | 0 .../openreplay/charts/assets/templates/ingress.yaml | 0 .../openreplay/charts/assets/templates/service.yaml | 0 .../charts/assets/templates/serviceaccount.yaml | 0 .../assets/templates/tests/test-connection.yaml | 0 .../openreplay/charts/assets/values.yaml | 0 .../openreplay/charts/chalice/.helmignore | 0 .../openreplay/charts/chalice/Chart.yaml | 0 .../openreplay/charts/chalice/templates/NOTES.txt | 0 .../charts/chalice/templates/_helpers.tpl | 0 .../charts/chalice/templates/deployment.yaml | 0 .../openreplay/charts/chalice/templates/hpa.yaml | 0 .../charts/chalice/templates/ingress.yaml | 0 .../charts/chalice/templates/service.yaml | 0 .../charts/chalice/templates/serviceaccount.yaml | 0 .../chalice/templates/tests/test-connection.yaml | 0 .../openreplay/charts/chalice/values.yaml | 0 .../openreplay/charts/db/.helmignore | 0 .../openreplay/charts/db/Chart.yaml | 0 .../openreplay/charts/db/templates/NOTES.txt | 0 .../openreplay/charts/db/templates/_helpers.tpl | 0 .../openreplay/charts/db/templates/deployment.yaml | 0 .../openreplay/charts/db/templates/hpa.yaml | 0 .../openreplay/charts/db/templates/ingress.yaml | 0 .../openreplay/charts/db/templates/service.yaml | 0 .../charts/db/templates/serviceaccount.yaml | 0 .../charts/db/templates/tests/test-connection.yaml | 0 .../openreplay/charts/db/values.yaml | 0 .../openreplay/charts/ender/.helmignore | 0 .../openreplay/charts/ender/Chart.yaml | 0 .../openreplay/charts/ender/templates/NOTES.txt | 0 .../openreplay/charts/ender/templates/_helpers.tpl | 0 .../charts/ender/templates/deployment.yaml | 0 .../openreplay/charts/ender/templates/hpa.yaml | 0 .../openreplay/charts/ender/templates/ingress.yaml | 0 .../openreplay/charts/ender/templates/service.yaml | 0 .../charts/ender/templates/serviceaccount.yaml | 0 .../ender/templates/tests/test-connection.yaml | 0 .../openreplay/charts/ender/values.yaml | 0 .../openreplay/charts/http/.helmignore | 0 .../openreplay/charts/http/Chart.yaml | 0 .../openreplay/charts/http/templates/NOTES.txt | 0 .../openreplay/charts/http/templates/_helpers.tpl | 0 .../charts/http/templates/deployment.yaml | 0 .../openreplay/charts/http/templates/hpa.yaml | 0 .../openreplay/charts/http/templates/ingress.yaml | 0 .../openreplay/charts/http/templates/service.yaml | 0 .../charts/http/templates/serviceaccount.yaml | 0 .../http/templates/tests/test-connection.yaml | 0 .../openreplay/charts/http/values.yaml | 0 .../openreplay/charts/integrations/.helmignore | 0 .../openreplay/charts/integrations/Chart.yaml | 0 .../charts/integrations/templates/NOTES.txt | 0 .../charts/integrations/templates/_helpers.tpl | 0 .../charts/integrations/templates/deployment.yaml | 0 .../charts/integrations/templates/hpa.yaml | 0 .../charts/integrations/templates/ingress.yaml | 0 .../charts/integrations/templates/service.yaml | 0 .../integrations/templates/serviceaccount.yaml | 0 .../templates/tests/test-connection.yaml | 0 .../openreplay/charts/integrations/values.yaml | 0 .../openreplay/charts/nginx-ingress/.helmignore | 0 .../openreplay/charts/nginx-ingress/Chart.yaml | 0 .../charts/nginx-ingress/templates/NOTES.txt | 0 .../charts/nginx-ingress/templates/_helpers.tpl | 0 .../charts/nginx-ingress/templates/configMap.yaml | 0 .../charts/nginx-ingress/templates/deployment.yaml | 0 .../charts/nginx-ingress/templates/hpa.yaml | 0 .../charts/nginx-ingress/templates/ingress.yaml | 0 .../charts/nginx-ingress/templates/secrets.yaml | 0 .../charts/nginx-ingress/templates/service.yaml | 0 .../nginx-ingress/templates/serviceaccount.yaml | 0 .../templates/tests/test-connection.yaml | 0 .../openreplay/charts/nginx-ingress/values.yaml | 0 .../openreplay/charts/sink/.helmignore | 0 .../openreplay/charts/sink/Chart.yaml | 0 .../openreplay/charts/sink/templates/NOTES.txt | 0 .../openreplay/charts/sink/templates/_helpers.tpl | 0 .../charts/sink/templates/deployment.yaml | 0 .../openreplay/charts/sink/templates/hpa.yaml | 0 .../openreplay/charts/sink/templates/ingress.yaml | 0 .../openreplay/charts/sink/templates/service.yaml | 0 .../charts/sink/templates/serviceaccount.yaml | 0 .../sink/templates/tests/test-connection.yaml | 0 .../openreplay/charts/sink/values.yaml | 0 .../openreplay/charts/storage/.helmignore | 0 .../openreplay/charts/storage/Chart.yaml | 0 .../openreplay/charts/storage/templates/NOTES.txt | 0 .../charts/storage/templates/_helpers.tpl | 0 .../charts/storage/templates/deployment.yaml | 0 .../openreplay/charts/storage/templates/hpa.yaml | 0 .../charts/storage/templates/ingress.yaml | 0 .../charts/storage/templates/service.yaml | 0 .../charts/storage/templates/serviceaccount.yaml | 0 .../storage/templates/tests/test-connection.yaml | 0 .../openreplay/charts/storage/values.yaml | 0 .../openreplay/charts/utilities/.helmignore | 0 .../openreplay/charts/utilities/Chart.yaml | 0 .../openreplay/charts/utilities/templates/NOTES.txt | 0 .../charts/utilities/templates/_helpers.tpl | 0 .../charts/utilities/templates/deployment.yaml | 0 .../openreplay/charts/utilities/templates/hpa.yaml | 0 .../charts/utilities/templates/ingress.yaml | 0 .../charts/utilities/templates/service.yaml | 0 .../charts/utilities/templates/serviceaccount.yaml | 0 .../utilities/templates/tests/test-connection.yaml | 0 .../openreplay/charts/utilities/values.yaml | 0 .../openreplay/files/clickhouse.sh | 0 .../{helm => helmcharts}/openreplay/files/dbops.sh | 0 .../{helm => helmcharts}/openreplay/files/kafka.sh | 0 .../{helm => helmcharts}/openreplay/files/minio.sh | 0 .../openreplay/files/postgresql.sh | 0 .../openreplay/templates/NOTES.txt | 0 .../openreplay/templates/_helpers.tpl | 0 .../openreplay/templates/job.yaml | 0 .../openreplay/templates/serviceaccount.yaml | 0 .../{helm => helmcharts}/openreplay/values.yaml | 0 scripts/helm/{helm => helmcharts}/vars.yaml | 0 scripts/helm/openreplay-cli | 2 +- 327 files changed, 1 insertion(+), 1 deletion(-) rename scripts/helm/{helm => helmcharts}/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/templates/statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/clickhouse/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/templates/svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/values-production.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/charts/zookeeper/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/files/jks/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/kafka.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/requirements.lock (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/requirements.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/extra-list.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/jaas-secret.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/jks-secret.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/jmx-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/jmx-metrics-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/kafka-metrics-deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/kafka-metrics-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/log4j-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/poddisruptionbudget.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/role.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/rolebinding.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/scripts-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/servicemonitor-metrics.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/svc-external-access.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/svc-headless.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/templates/svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/values-production.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/kafka/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/ci/values-production.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/deployment-standalone.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/networkpolicy.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/pvc-standalone.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/secrets.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/svc-headless.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/templates/tls-secrets.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/values-production.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/minio/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_affinities.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_capabilities.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_errors.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_images.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_labels.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_names.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_secrets.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_storage.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_utils.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_validations.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/templates/_warnings.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/charts/common/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/ci/commonAnnotations.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/ci/default-values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/files/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/files/conf.d/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/postgresql.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/requirements.lock (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/requirements.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/extended-config-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/extra-list.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/initialization-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/metrics-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/metrics-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/networkpolicy.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/podsecuritypolicy.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/prometheusrule.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/role.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/rolebinding.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/secrets.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/servicemonitor.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/statefulset-slaves.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/svc-headless.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/svc-read.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/templates/svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/values-production.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/values.schema.json (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/postgresql/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/Chart.lock (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/README.md (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_affinities.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_capabilities.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_errors.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_images.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_ingress.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_labels.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_names.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_secrets.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_storage.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_tplvalues.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_utils.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/_warnings.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/validations/_redis.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/templates/validations/_validations.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/charts/common/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/ci/default-values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/ci/extra-flags-values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/ci/production-sentinel-values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/img/redis-cluster-topology.png (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/img/redis-topology.png (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/configmap-scripts.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/headless-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/health-configmap.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/metrics-prometheus.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/metrics-sentinel-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/metrics-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/networkpolicy.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/pdb.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/prometheusrule.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/psp.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-master-statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-master-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-node-statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-role.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-rolebinding.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-slave-statefulset.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-slave-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/redis-with-sentinel-svc.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/templates/secret.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/values.schema.json (100%) rename scripts/helm/{helm => helmcharts}/databases/charts/redis/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/databases/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/databases/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/databases/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/init.sh (100%) rename scripts/helm/{helm => helmcharts}/openreplay/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/alerts/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/assets/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/chalice/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/db/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/ender/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/http/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/integrations/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/configMap.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/secrets.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/nginx-ingress/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/sink/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/storage/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/.helmignore (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/Chart.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/deployment.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/hpa.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/ingress.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/service.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/templates/tests/test-connection.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/charts/utilities/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/files/clickhouse.sh (100%) rename scripts/helm/{helm => helmcharts}/openreplay/files/dbops.sh (100%) rename scripts/helm/{helm => helmcharts}/openreplay/files/kafka.sh (100%) rename scripts/helm/{helm => helmcharts}/openreplay/files/minio.sh (100%) rename scripts/helm/{helm => helmcharts}/openreplay/files/postgresql.sh (100%) rename scripts/helm/{helm => helmcharts}/openreplay/templates/NOTES.txt (100%) rename scripts/helm/{helm => helmcharts}/openreplay/templates/_helpers.tpl (100%) rename scripts/helm/{helm => helmcharts}/openreplay/templates/job.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/templates/serviceaccount.yaml (100%) rename scripts/helm/{helm => helmcharts}/openreplay/values.yaml (100%) rename scripts/helm/{helm => helmcharts}/vars.yaml (100%) diff --git a/scripts/helm/helm/README.md b/scripts/helm/helmcharts/README.md similarity index 100% rename from scripts/helm/helm/README.md rename to scripts/helm/helmcharts/README.md diff --git a/scripts/helm/helm/databases/.helmignore b/scripts/helm/helmcharts/databases/.helmignore similarity index 100% rename from scripts/helm/helm/databases/.helmignore rename to scripts/helm/helmcharts/databases/.helmignore diff --git a/scripts/helm/helm/databases/Chart.yaml b/scripts/helm/helmcharts/databases/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/Chart.yaml rename to scripts/helm/helmcharts/databases/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/clickhouse/.helmignore b/scripts/helm/helmcharts/databases/charts/clickhouse/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/.helmignore rename to scripts/helm/helmcharts/databases/charts/clickhouse/.helmignore diff --git a/scripts/helm/helm/databases/charts/clickhouse/Chart.yaml b/scripts/helm/helmcharts/databases/charts/clickhouse/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/clickhouse/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml b/scripts/helm/helmcharts/databases/charts/clickhouse/templates/service.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/templates/service.yaml rename to scripts/helm/helmcharts/databases/charts/clickhouse/templates/service.yaml diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml b/scripts/helm/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml b/scripts/helm/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/templates/statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/clickhouse/values.yaml b/scripts/helm/helmcharts/databases/charts/clickhouse/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/clickhouse/values.yaml rename to scripts/helm/helmcharts/databases/charts/clickhouse/values.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/.helmignore b/scripts/helm/helmcharts/databases/charts/kafka/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/.helmignore rename to scripts/helm/helmcharts/databases/charts/kafka/.helmignore diff --git a/scripts/helm/helm/databases/charts/kafka/Chart.yaml b/scripts/helm/helmcharts/databases/charts/kafka/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/README.md b/scripts/helm/helmcharts/databases/charts/kafka/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/README.md rename to scripts/helm/helmcharts/databases/charts/kafka/README.md diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/.helmignore rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/README.md rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/README.md diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/templates/svc.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values-production.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml b/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/charts/zookeeper/values.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/files/jks/README.md b/scripts/helm/helmcharts/databases/charts/kafka/files/jks/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/files/jks/README.md rename to scripts/helm/helmcharts/databases/charts/kafka/files/jks/README.md diff --git a/scripts/helm/helm/databases/charts/kafka/kafka.yaml b/scripts/helm/helmcharts/databases/charts/kafka/kafka.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/kafka.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/kafka.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/requirements.lock b/scripts/helm/helmcharts/databases/charts/kafka/requirements.lock similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/requirements.lock rename to scripts/helm/helmcharts/databases/charts/kafka/requirements.lock diff --git a/scripts/helm/helm/databases/charts/kafka/requirements.yaml b/scripts/helm/helmcharts/databases/charts/kafka/requirements.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/requirements.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/requirements.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt b/scripts/helm/helmcharts/databases/charts/kafka/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/NOTES.txt rename to scripts/helm/helmcharts/databases/charts/kafka/templates/NOTES.txt diff --git a/scripts/helm/helm/databases/charts/kafka/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/charts/kafka/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/charts/kafka/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/charts/kafka/templates/configmap.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/configmap.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/configmap.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/extra-list.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/extra-list.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/extra-list.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/extra-list.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/jaas-secret.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/jaas-secret.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/jks-secret.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/jks-secret.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/jks-secret.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/jks-secret.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/jmx-configmap.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/jmx-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/jmx-metrics-svc.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-deployment.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/kafka-metrics-svc.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/log4j-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/poddisruptionbudget.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/role.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/role.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/role.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/role.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/rolebinding.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/rolebinding.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/rolebinding.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/scripts-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/servicemonitor-metrics.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/svc-external-access.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/svc-headless.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/svc-headless.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/templates/svc.yaml b/scripts/helm/helmcharts/databases/charts/kafka/templates/svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/templates/svc.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/templates/svc.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/values-production.yaml b/scripts/helm/helmcharts/databases/charts/kafka/values-production.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/values-production.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/values-production.yaml diff --git a/scripts/helm/helm/databases/charts/kafka/values.yaml b/scripts/helm/helmcharts/databases/charts/kafka/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/kafka/values.yaml rename to scripts/helm/helmcharts/databases/charts/kafka/values.yaml diff --git a/scripts/helm/helm/databases/charts/minio/.helmignore b/scripts/helm/helmcharts/databases/charts/minio/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/minio/.helmignore rename to scripts/helm/helmcharts/databases/charts/minio/.helmignore diff --git a/scripts/helm/helm/databases/charts/minio/Chart.yaml b/scripts/helm/helmcharts/databases/charts/minio/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/minio/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/minio/README.md b/scripts/helm/helmcharts/databases/charts/minio/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/minio/README.md rename to scripts/helm/helmcharts/databases/charts/minio/README.md diff --git a/scripts/helm/helm/databases/charts/minio/ci/values-production.yaml b/scripts/helm/helmcharts/databases/charts/minio/ci/values-production.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/ci/values-production.yaml rename to scripts/helm/helmcharts/databases/charts/minio/ci/values-production.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/NOTES.txt b/scripts/helm/helmcharts/databases/charts/minio/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/NOTES.txt rename to scripts/helm/helmcharts/databases/charts/minio/templates/NOTES.txt diff --git a/scripts/helm/helm/databases/charts/minio/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/charts/minio/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/charts/minio/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/charts/minio/templates/deployment-standalone.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/deployment-standalone.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/ingress.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/ingress.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/ingress.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/networkpolicy.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/networkpolicy.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/networkpolicy.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/pvc-standalone.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/pvc-standalone.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/secrets.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/secrets.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/secrets.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/secrets.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/service.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/service.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/service.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/service.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/serviceaccount.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/statefulset.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/svc-headless.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/svc-headless.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/svc-headless.yaml diff --git a/scripts/helm/helm/databases/charts/minio/templates/tls-secrets.yaml b/scripts/helm/helmcharts/databases/charts/minio/templates/tls-secrets.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/templates/tls-secrets.yaml rename to scripts/helm/helmcharts/databases/charts/minio/templates/tls-secrets.yaml diff --git a/scripts/helm/helm/databases/charts/minio/values-production.yaml b/scripts/helm/helmcharts/databases/charts/minio/values-production.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/values-production.yaml rename to scripts/helm/helmcharts/databases/charts/minio/values-production.yaml diff --git a/scripts/helm/helm/databases/charts/minio/values.yaml b/scripts/helm/helmcharts/databases/charts/minio/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/minio/values.yaml rename to scripts/helm/helmcharts/databases/charts/minio/values.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/.helmignore b/scripts/helm/helmcharts/databases/charts/postgresql/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/.helmignore rename to scripts/helm/helmcharts/databases/charts/postgresql/.helmignore diff --git a/scripts/helm/helm/databases/charts/postgresql/Chart.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/README.md b/scripts/helm/helmcharts/databases/charts/postgresql/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/README.md rename to scripts/helm/helmcharts/databases/charts/postgresql/README.md diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/.helmignore b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/.helmignore rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/.helmignore diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/Chart.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/README.md b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/README.md rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/README.md diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_affinities.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_affinities.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_capabilities.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_capabilities.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_errors.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_errors.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_images.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_images.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_labels.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_labels.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_names.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_names.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_secrets.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_secrets.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_storage.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_storage.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_utils.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_utils.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_validations.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_validations.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_warnings.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/templates/_warnings.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/charts/common/values.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/charts/common/values.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/charts/common/values.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/ci/commonAnnotations.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/ci/commonAnnotations.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/ci/default-values.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/ci/default-values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/ci/default-values.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/ci/default-values.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/files/README.md b/scripts/helm/helmcharts/databases/charts/postgresql/files/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/files/README.md rename to scripts/helm/helmcharts/databases/charts/postgresql/files/README.md diff --git a/scripts/helm/helm/databases/charts/postgresql/files/conf.d/README.md b/scripts/helm/helmcharts/databases/charts/postgresql/files/conf.d/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/files/conf.d/README.md rename to scripts/helm/helmcharts/databases/charts/postgresql/files/conf.d/README.md diff --git a/scripts/helm/helm/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md b/scripts/helm/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md rename to scripts/helm/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md diff --git a/scripts/helm/helm/databases/charts/postgresql/postgresql.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/postgresql.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/postgresql.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/postgresql.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/requirements.lock b/scripts/helm/helmcharts/databases/charts/postgresql/requirements.lock similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/requirements.lock rename to scripts/helm/helmcharts/databases/charts/postgresql/requirements.lock diff --git a/scripts/helm/helm/databases/charts/postgresql/requirements.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/requirements.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/requirements.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/requirements.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/NOTES.txt b/scripts/helm/helmcharts/databases/charts/postgresql/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/NOTES.txt rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/NOTES.txt diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/charts/postgresql/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/configmap.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/configmap.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/configmap.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/extended-config-configmap.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/extended-config-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/extra-list.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/extra-list.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/extra-list.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/extra-list.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/initialization-configmap.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/initialization-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/metrics-configmap.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/metrics-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/metrics-svc.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/metrics-svc.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/networkpolicy.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/networkpolicy.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/podsecuritypolicy.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/podsecuritypolicy.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/prometheusrule.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/prometheusrule.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/role.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/role.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/role.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/role.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/rolebinding.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/rolebinding.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/secrets.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/secrets.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/secrets.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/secrets.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/serviceaccount.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/servicemonitor.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/servicemonitor.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/statefulset-slaves.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/statefulset-slaves.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/statefulset.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/svc-headless.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/svc-headless.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/svc-read.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-read.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/svc-read.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-read.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/templates/svc.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/templates/svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/templates/svc.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/templates/svc.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/values-production.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/values-production.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/values-production.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/values-production.yaml diff --git a/scripts/helm/helm/databases/charts/postgresql/values.schema.json b/scripts/helm/helmcharts/databases/charts/postgresql/values.schema.json similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/values.schema.json rename to scripts/helm/helmcharts/databases/charts/postgresql/values.schema.json diff --git a/scripts/helm/helm/databases/charts/postgresql/values.yaml b/scripts/helm/helmcharts/databases/charts/postgresql/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/postgresql/values.yaml rename to scripts/helm/helmcharts/databases/charts/postgresql/values.yaml diff --git a/scripts/helm/helm/databases/charts/redis/.helmignore b/scripts/helm/helmcharts/databases/charts/redis/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/redis/.helmignore rename to scripts/helm/helmcharts/databases/charts/redis/.helmignore diff --git a/scripts/helm/helm/databases/charts/redis/Chart.lock b/scripts/helm/helmcharts/databases/charts/redis/Chart.lock similarity index 100% rename from scripts/helm/helm/databases/charts/redis/Chart.lock rename to scripts/helm/helmcharts/databases/charts/redis/Chart.lock diff --git a/scripts/helm/helm/databases/charts/redis/Chart.yaml b/scripts/helm/helmcharts/databases/charts/redis/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/redis/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/redis/README.md b/scripts/helm/helmcharts/databases/charts/redis/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/redis/README.md rename to scripts/helm/helmcharts/databases/charts/redis/README.md diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/.helmignore b/scripts/helm/helmcharts/databases/charts/redis/charts/common/.helmignore similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/.helmignore rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/.helmignore diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/Chart.yaml b/scripts/helm/helmcharts/databases/charts/redis/charts/common/Chart.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/Chart.yaml rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/Chart.yaml diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/README.md b/scripts/helm/helmcharts/databases/charts/redis/charts/common/README.md similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/README.md rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/README.md diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_affinities.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_affinities.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_capabilities.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_capabilities.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_errors.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_errors.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_images.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_images.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_ingress.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_ingress.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_labels.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_labels.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_names.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_names.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_secrets.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_secrets.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_storage.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_storage.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_tplvalues.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_tplvalues.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_utils.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_utils.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/_warnings.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/_warnings.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_redis.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_redis.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_validations.tpl b/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/templates/validations/_validations.tpl rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl diff --git a/scripts/helm/helm/databases/charts/redis/charts/common/values.yaml b/scripts/helm/helmcharts/databases/charts/redis/charts/common/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/charts/common/values.yaml rename to scripts/helm/helmcharts/databases/charts/redis/charts/common/values.yaml diff --git a/scripts/helm/helm/databases/charts/redis/ci/default-values.yaml b/scripts/helm/helmcharts/databases/charts/redis/ci/default-values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/ci/default-values.yaml rename to scripts/helm/helmcharts/databases/charts/redis/ci/default-values.yaml diff --git a/scripts/helm/helm/databases/charts/redis/ci/extra-flags-values.yaml b/scripts/helm/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/ci/extra-flags-values.yaml rename to scripts/helm/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml diff --git a/scripts/helm/helm/databases/charts/redis/ci/production-sentinel-values.yaml b/scripts/helm/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/ci/production-sentinel-values.yaml rename to scripts/helm/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml diff --git a/scripts/helm/helm/databases/charts/redis/img/redis-cluster-topology.png b/scripts/helm/helmcharts/databases/charts/redis/img/redis-cluster-topology.png similarity index 100% rename from scripts/helm/helm/databases/charts/redis/img/redis-cluster-topology.png rename to scripts/helm/helmcharts/databases/charts/redis/img/redis-cluster-topology.png diff --git a/scripts/helm/helm/databases/charts/redis/img/redis-topology.png b/scripts/helm/helmcharts/databases/charts/redis/img/redis-topology.png similarity index 100% rename from scripts/helm/helm/databases/charts/redis/img/redis-topology.png rename to scripts/helm/helmcharts/databases/charts/redis/img/redis-topology.png diff --git a/scripts/helm/helm/databases/charts/redis/templates/NOTES.txt b/scripts/helm/helmcharts/databases/charts/redis/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/NOTES.txt rename to scripts/helm/helmcharts/databases/charts/redis/templates/NOTES.txt diff --git a/scripts/helm/helm/databases/charts/redis/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/charts/redis/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/charts/redis/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/charts/redis/templates/configmap-scripts.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/configmap-scripts.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/configmap.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/configmap.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/configmap.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/headless-svc.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/headless-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/headless-svc.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/headless-svc.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/health-configmap.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/health-configmap.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/health-configmap.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/health-configmap.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-prometheus.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/metrics-prometheus.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-svc.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/metrics-sentinel-svc.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/metrics-svc.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/metrics-svc.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/metrics-svc.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/networkpolicy.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/networkpolicy.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/networkpolicy.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/pdb.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/pdb.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/pdb.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/pdb.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/prometheusrule.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/prometheusrule.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/prometheusrule.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/prometheusrule.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/psp.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/psp.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/psp.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/psp.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-master-statefulset.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-master-statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-master-svc.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-master-svc.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-node-statefulset.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-node-statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-role.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-role.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-role.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-role.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-rolebinding.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-rolebinding.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-serviceaccount.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-serviceaccount.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-slave-statefulset.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-slave-statefulset.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-slave-svc.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-slave-svc.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/redis-with-sentinel-svc.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/redis-with-sentinel-svc.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml diff --git a/scripts/helm/helm/databases/charts/redis/templates/secret.yaml b/scripts/helm/helmcharts/databases/charts/redis/templates/secret.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/templates/secret.yaml rename to scripts/helm/helmcharts/databases/charts/redis/templates/secret.yaml diff --git a/scripts/helm/helm/databases/charts/redis/values.schema.json b/scripts/helm/helmcharts/databases/charts/redis/values.schema.json similarity index 100% rename from scripts/helm/helm/databases/charts/redis/values.schema.json rename to scripts/helm/helmcharts/databases/charts/redis/values.schema.json diff --git a/scripts/helm/helm/databases/charts/redis/values.yaml b/scripts/helm/helmcharts/databases/charts/redis/values.yaml similarity index 100% rename from scripts/helm/helm/databases/charts/redis/values.yaml rename to scripts/helm/helmcharts/databases/charts/redis/values.yaml diff --git a/scripts/helm/helm/databases/templates/NOTES.txt b/scripts/helm/helmcharts/databases/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/databases/templates/NOTES.txt rename to scripts/helm/helmcharts/databases/templates/NOTES.txt diff --git a/scripts/helm/helm/databases/templates/_helpers.tpl b/scripts/helm/helmcharts/databases/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/databases/templates/_helpers.tpl rename to scripts/helm/helmcharts/databases/templates/_helpers.tpl diff --git a/scripts/helm/helm/databases/values.yaml b/scripts/helm/helmcharts/databases/values.yaml similarity index 100% rename from scripts/helm/helm/databases/values.yaml rename to scripts/helm/helmcharts/databases/values.yaml diff --git a/scripts/helm/helm/init.sh b/scripts/helm/helmcharts/init.sh similarity index 100% rename from scripts/helm/helm/init.sh rename to scripts/helm/helmcharts/init.sh diff --git a/scripts/helm/helm/openreplay/.helmignore b/scripts/helm/helmcharts/openreplay/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/.helmignore rename to scripts/helm/helmcharts/openreplay/.helmignore diff --git a/scripts/helm/helm/openreplay/Chart.yaml b/scripts/helm/helmcharts/openreplay/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/Chart.yaml rename to scripts/helm/helmcharts/openreplay/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/.helmignore b/scripts/helm/helmcharts/openreplay/charts/alerts/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/alerts/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/alerts/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/alerts/values.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/alerts/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/alerts/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/.helmignore b/scripts/helm/helmcharts/openreplay/charts/assets/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/assets/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/assets/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/assets/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/assets/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/assets/values.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/assets/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/assets/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/.helmignore b/scripts/helm/helmcharts/openreplay/charts/chalice/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/chalice/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/chalice/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/chalice/values.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/chalice/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/.helmignore b/scripts/helm/helmcharts/openreplay/charts/db/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/db/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/db/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/db/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/db/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/db/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/db/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/db/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/db/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/db/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/db/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/db/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/db/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/db/values.yaml b/scripts/helm/helmcharts/openreplay/charts/db/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/db/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/db/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/.helmignore b/scripts/helm/helmcharts/openreplay/charts/ender/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/ender/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/ender/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/ender/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/ender/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/ender/values.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/ender/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/ender/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/.helmignore b/scripts/helm/helmcharts/openreplay/charts/http/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/http/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/http/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/http/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/http/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/http/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/http/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/http/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/http/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/http/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/http/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/http/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/http/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/http/values.yaml b/scripts/helm/helmcharts/openreplay/charts/http/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/http/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/http/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/.helmignore b/scripts/helm/helmcharts/openreplay/charts/integrations/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/integrations/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/integrations/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/integrations/values.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/integrations/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/integrations/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/.helmignore b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/configMap.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/secrets.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/nginx-ingress/values.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/nginx-ingress/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/nginx-ingress/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/.helmignore b/scripts/helm/helmcharts/openreplay/charts/sink/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/sink/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/sink/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/sink/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/sink/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/sink/values.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/sink/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/sink/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/.helmignore b/scripts/helm/helmcharts/openreplay/charts/storage/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/storage/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/storage/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/storage/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/storage/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/storage/values.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/storage/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/storage/values.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/.helmignore b/scripts/helm/helmcharts/openreplay/charts/utilities/.helmignore similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/.helmignore rename to scripts/helm/helmcharts/openreplay/charts/utilities/.helmignore diff --git a/scripts/helm/helm/openreplay/charts/utilities/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/Chart.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/deployment.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/deployment.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/deployment.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/hpa.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/hpa.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/hpa.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/hpa.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/ingress.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/ingress.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/ingress.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/ingress.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/service.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/service.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/service.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/service.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/templates/tests/test-connection.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/templates/tests/test-connection.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml diff --git a/scripts/helm/helm/openreplay/charts/utilities/values.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/charts/utilities/values.yaml rename to scripts/helm/helmcharts/openreplay/charts/utilities/values.yaml diff --git a/scripts/helm/helm/openreplay/files/clickhouse.sh b/scripts/helm/helmcharts/openreplay/files/clickhouse.sh similarity index 100% rename from scripts/helm/helm/openreplay/files/clickhouse.sh rename to scripts/helm/helmcharts/openreplay/files/clickhouse.sh diff --git a/scripts/helm/helm/openreplay/files/dbops.sh b/scripts/helm/helmcharts/openreplay/files/dbops.sh similarity index 100% rename from scripts/helm/helm/openreplay/files/dbops.sh rename to scripts/helm/helmcharts/openreplay/files/dbops.sh diff --git a/scripts/helm/helm/openreplay/files/kafka.sh b/scripts/helm/helmcharts/openreplay/files/kafka.sh similarity index 100% rename from scripts/helm/helm/openreplay/files/kafka.sh rename to scripts/helm/helmcharts/openreplay/files/kafka.sh diff --git a/scripts/helm/helm/openreplay/files/minio.sh b/scripts/helm/helmcharts/openreplay/files/minio.sh similarity index 100% rename from scripts/helm/helm/openreplay/files/minio.sh rename to scripts/helm/helmcharts/openreplay/files/minio.sh diff --git a/scripts/helm/helm/openreplay/files/postgresql.sh b/scripts/helm/helmcharts/openreplay/files/postgresql.sh similarity index 100% rename from scripts/helm/helm/openreplay/files/postgresql.sh rename to scripts/helm/helmcharts/openreplay/files/postgresql.sh diff --git a/scripts/helm/helm/openreplay/templates/NOTES.txt b/scripts/helm/helmcharts/openreplay/templates/NOTES.txt similarity index 100% rename from scripts/helm/helm/openreplay/templates/NOTES.txt rename to scripts/helm/helmcharts/openreplay/templates/NOTES.txt diff --git a/scripts/helm/helm/openreplay/templates/_helpers.tpl b/scripts/helm/helmcharts/openreplay/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helm/openreplay/templates/_helpers.tpl rename to scripts/helm/helmcharts/openreplay/templates/_helpers.tpl diff --git a/scripts/helm/helm/openreplay/templates/job.yaml b/scripts/helm/helmcharts/openreplay/templates/job.yaml similarity index 100% rename from scripts/helm/helm/openreplay/templates/job.yaml rename to scripts/helm/helmcharts/openreplay/templates/job.yaml diff --git a/scripts/helm/helm/openreplay/templates/serviceaccount.yaml b/scripts/helm/helmcharts/openreplay/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helm/openreplay/templates/serviceaccount.yaml rename to scripts/helm/helmcharts/openreplay/templates/serviceaccount.yaml diff --git a/scripts/helm/helm/openreplay/values.yaml b/scripts/helm/helmcharts/openreplay/values.yaml similarity index 100% rename from scripts/helm/helm/openreplay/values.yaml rename to scripts/helm/helmcharts/openreplay/values.yaml diff --git a/scripts/helm/helm/vars.yaml b/scripts/helm/helmcharts/vars.yaml similarity index 100% rename from scripts/helm/helm/vars.yaml rename to scripts/helm/helmcharts/vars.yaml diff --git a/scripts/helm/openreplay-cli b/scripts/helm/openreplay-cli index 6b6fae514..0f229e93a 100755 --- a/scripts/helm/openreplay-cli +++ b/scripts/helm/openreplay-cli @@ -109,7 +109,7 @@ legacyInstall() { } helmInstall() { - helm upgrade --install openreplay helmv2/openreplay -f helmv2/vars.yaml + helm upgrade --install openreplay helmcharts/openreplay -f helmcharts/vars.yaml } upgrade() { From 6c391c89ea9317325ca989cadf96cc697a7914bc Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 01:21:55 +0530 Subject: [PATCH 109/206] chore(helm): upgrade postgresql version Signed-off-by: Rajesh Rajendran --- scripts/helm/helmcharts/databases/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helm/helmcharts/databases/values.yaml b/scripts/helm/helmcharts/databases/values.yaml index 13b52cd60..4e8af5e10 100644 --- a/scripts/helm/helmcharts/databases/values.yaml +++ b/scripts/helm/helmcharts/databases/values.yaml @@ -103,7 +103,7 @@ postgresql: # postgresqlPassword: asayerPostgres fullnameOverride: postgresql image: - tag: 13.3.0-debian-10-r53 + tag: 13.5.0-debian-10-r62 resources: limits: cpu: 1 From 97968ab95e26157af480122f616dc83ddd6a3050 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 01:28:20 +0530 Subject: [PATCH 110/206] chore(helm): update the chalice version Signed-off-by: Rajesh Rajendran --- scripts/helm/helmcharts/databases/values.yaml | 2 ++ scripts/helm/helmcharts/openreplay/templates/job.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/helm/helmcharts/databases/values.yaml b/scripts/helm/helmcharts/databases/values.yaml index 4e8af5e10..6c7f28960 100644 --- a/scripts/helm/helmcharts/databases/values.yaml +++ b/scripts/helm/helmcharts/databases/values.yaml @@ -133,6 +133,8 @@ kafka: # Enterprise dbs clickhouse: + image: + tag: "21.9.4.35" enabled: false postgreql: diff --git a/scripts/helm/helmcharts/openreplay/templates/job.yaml b/scripts/helm/helmcharts/openreplay/templates/job.yaml index 086d3dd24..aa1b4467d 100644 --- a/scripts/helm/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helm/helmcharts/openreplay/templates/job.yaml @@ -97,7 +97,7 @@ spec: {{- if .Values.global.enterpriseEditionLicense }} # Enterprise migration - name: clickhouse - image: yandex/clickhouse-client:20.9 + image: yandex/clickhouse-client:21.9.4.35 env: - name: PREVIOUS_APP_VERSION value: "{{ .Values.fromVersion }}" From 88ee70007159a206dd6358858f78bd3a02196527 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 01:44:20 +0530 Subject: [PATCH 111/206] chore(helm): updating openreplay version --- frontend/env.js | 2 +- scripts/helm/helmcharts/init.sh | 4 ++-- scripts/helm/helmcharts/openreplay/Chart.yaml | 2 +- .../helmcharts/openreplay/charts/alerts/Chart.yaml | 2 +- .../helmcharts/openreplay/charts/assets/Chart.yaml | 2 +- .../helmcharts/openreplay/charts/chalice/Chart.yaml | 2 +- .../helm/helmcharts/openreplay/charts/db/Chart.yaml | 2 +- .../helm/helmcharts/openreplay/charts/ender/Chart.yaml | 2 +- .../helm/helmcharts/openreplay/charts/http/Chart.yaml | 2 +- .../openreplay/charts/integrations/Chart.yaml | 2 +- .../openreplay/charts/nginx-ingress/Chart.yaml | 2 +- .../helm/helmcharts/openreplay/charts/sink/Chart.yaml | 2 +- .../helmcharts/openreplay/charts/storage/Chart.yaml | 2 +- .../helmcharts/openreplay/charts/utilities/Chart.yaml | 2 +- scripts/helm/helmcharts/vars.yaml | 2 +- scripts/helm/helmcharts/versionUpdater.sh | 10 ++++++++++ 16 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 scripts/helm/helmcharts/versionUpdater.sh diff --git a/frontend/env.js b/frontend/env.js index f7c49c874..10513c3b1 100644 --- a/frontend/env.js +++ b/frontend/env.js @@ -13,7 +13,7 @@ const oss = { ORIGIN: () => 'window.location.origin', API_EDP: () => 'window.location.origin + "/api"', ASSETS_HOST: () => 'window.location.origin + "/assets"', - VERSION: '1.3.6', + VERSION: '1.4.0', SOURCEMAP: true, MINIO_ENDPOINT: process.env.MINIO_ENDPOINT, MINIO_PORT: process.env.MINIO_PORT, diff --git a/scripts/helm/helmcharts/init.sh b/scripts/helm/helmcharts/init.sh index 8b926cf50..c284fab92 100644 --- a/scripts/helm/helmcharts/init.sh +++ b/scripts/helm/helmcharts/init.sh @@ -15,7 +15,7 @@ fatal() exit 1 } -version=${OPENREPLAY_VERSION:-'v1.4.0'} +version="v1.4.0" usr=`whoami` # Installing k3s @@ -76,7 +76,7 @@ randomPass() { }|| { info " Downloading vars file" -curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${version}/scripts/helm/helm/vars.yaml +curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${version="v1.4.0" } diff --git a/scripts/helm/helmcharts/openreplay/Chart.yaml b/scripts/helm/helmcharts/openreplay/Chart.yaml index 70783afea..d7bfd0074 100644 --- a/scripts/helm/helmcharts/openreplay/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/Chart.yaml @@ -22,4 +22,4 @@ version: 0.1.0 # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. # Ref: https://github.com/helm/helm/issues/7858#issuecomment-608114589 -AppVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml index 101e24ac1..4cda7945b 100644 --- a/scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml index 924e9d634..fe932b71c 100644 --- a/scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml index 4516f0b9b..42ec55dec 100644 --- a/scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml index 816ccccb9..af2db8c06 100644 --- a/scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml index 0e3fadd03..92e286a7d 100644 --- a/scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml index 54a567e21..91b9ce66f 100644 --- a/scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml index 48fffed45..a643daa69 100644 --- a/scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml index 0019ec4f2..3c824500e 100644 --- a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml index ac86f8910..2b9f71d56 100644 --- a/scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml index 3d122333f..329af9ad0 100644 --- a/scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml b/scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml index 062a1ea73..4c00e90b2 100644 --- a/scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.3.6" +AppVersion: "v1.4.0" diff --git a/scripts/helm/helmcharts/vars.yaml b/scripts/helm/helmcharts/vars.yaml index f15033aa3..47ec01afe 100644 --- a/scripts/helm/helmcharts/vars.yaml +++ b/scripts/helm/helmcharts/vars.yaml @@ -1,4 +1,4 @@ -fromVersion: "v1.3.6" +fromVersion: "v1.4.0" # Databases specific variables postgresql: &postgres # For genrating psswords diff --git a/scripts/helm/helmcharts/versionUpdater.sh b/scripts/helm/helmcharts/versionUpdater.sh new file mode 100644 index 000000000..5fc8230a2 --- /dev/null +++ b/scripts/helm/helmcharts/versionUpdater.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# This script will update the version of openreplay components. +currentVersion=$1 +[[ -z $currentVersion ]] && { + echo "Usage: $0 " + echo "eg: $0 v1.5.0" +} +find ./openreplay -type f -iname chart.yaml -exec sed -i "s/AppVersion.*/AppVersion: \"$currentVersion\"/g" {} \; +sed -i "s/fromVersion.*/fromVersion: \"$currentVersion\"/g" vars.yaml +sed -i "s/version.*/version=\"$currentVersion\"/g" init.sh From b4a1d8344e55378baa5a6587e9ff43b29f60076c Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 01:58:04 +0530 Subject: [PATCH 112/206] chore(helm): dynamic injection of version in chalice Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/chalice/templates/deployment.yaml | 2 ++ scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index a092c29aa..805873de7 100644 --- a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -38,6 +38,8 @@ spec: {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: version_number + value: '{{ .Chart.AppVersion }}' - name: pg_host value: '{{ .Values.global.postgresql.postgresqlHost }}' - name: pg_port diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml index bb8139ede..509516136 100644 --- a/scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml @@ -98,7 +98,6 @@ env: # Enable logging for python app # Ref: https://stackoverflow.com/questions/43969743/logs-in-kubernetes-pod-not-showing-up PYTHONUNBUFFERED: '0' - version_number: '1.3.6' SAML2_MD_URL: '' idp_entityId: '' idp_sso_url: '' From f2003d1a8b51ab291ccf384cfa5f90b4ce1f688f Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 01:59:18 +0530 Subject: [PATCH 113/206] chore(helm): update default values Signed-off-by: Rajesh Rajendran --- scripts/helm/helmcharts/vars.yaml | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/scripts/helm/helmcharts/vars.yaml b/scripts/helm/helmcharts/vars.yaml index 47ec01afe..ebf9888cb 100644 --- a/scripts/helm/helmcharts/vars.yaml +++ b/scripts/helm/helmcharts/vars.yaml @@ -49,19 +49,28 @@ global: accessKey: "changeMeMinioAccessKey" secretKey: "changeMeMinioPassword" - chalice: - env: - jwt_secret: "SetARandomStringHere" - # EMAIL_HOST: '' - # EMAIL_PORT: '587' - # EMAIL_USER: '' - # EMAIL_PASSWORD: '' - # EMAIL_USE_TLS: 'true' - # EMAIL_FROM: OpenReplay - enterpriseEditionLicense: "" domainName: "" +chalice: + env: + jwt_secret: "SetARandomStringHere" + # EMAIL_HOST: '' + # EMAIL_PORT: '587' + # EMAIL_USER: '' + # EMAIL_PASSWORD: '' + # EMAIL_USE_TLS: 'true' + # EMAIL_USE_SSL: 'false' + # EMAIL_SSL_KEY: '' + # EMAIL_SSL_CERT: '' + # EMAIL_FROM: OpenReplay + # SAML2_MD_URL: '' + # idp_entityId: '' + # idp_sso_url: '' + # idp_x509cert: '' + # idp_sls_url: '' + # idp_name: '' + # If you want to override something # chartname: From 39ae351b6b47bd8b33d435dab73d5a1cd9cfbc45 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 02:10:31 +0530 Subject: [PATCH 114/206] chore(structure): changed folder structure for helm charts Signed-off-by: Rajesh Rajendran --- scripts/helm/openreplay-cli | 2 +- scripts/{helm => }/helmcharts/README.md | 0 scripts/{helm => }/helmcharts/databases/.helmignore | 0 scripts/{helm => }/helmcharts/databases/Chart.yaml | 0 .../databases/charts/clickhouse/.helmignore | 0 .../databases/charts/clickhouse/Chart.yaml | 0 .../charts/clickhouse/templates/_helpers.tpl | 0 .../charts/clickhouse/templates/service.yaml | 0 .../charts/clickhouse/templates/serviceaccount.yaml | 0 .../charts/clickhouse/templates/statefulset.yaml | 0 .../databases/charts/clickhouse/values.yaml | 0 .../helmcharts/databases/charts/kafka/.helmignore | 0 .../helmcharts/databases/charts/kafka/Chart.yaml | 0 .../helmcharts/databases/charts/kafka/README.md | 0 .../charts/kafka/charts/zookeeper/.helmignore | 0 .../charts/kafka/charts/zookeeper/Chart.yaml | 0 .../charts/kafka/charts/zookeeper/README.md | 0 .../kafka/charts/zookeeper/templates/NOTES.txt | 0 .../kafka/charts/zookeeper/templates/_helpers.tpl | 0 .../kafka/charts/zookeeper/templates/configmap.yaml | 0 .../charts/zookeeper/templates/metrics-svc.yaml | 0 .../charts/zookeeper/templates/networkpolicy.yaml | 0 .../zookeeper/templates/poddisruptionbudget.yaml | 0 .../charts/zookeeper/templates/prometheusrules.yaml | 0 .../kafka/charts/zookeeper/templates/secrets.yaml | 0 .../charts/zookeeper/templates/serviceaccount.yaml | 0 .../charts/zookeeper/templates/servicemonitor.yaml | 0 .../charts/zookeeper/templates/statefulset.yaml | 0 .../charts/zookeeper/templates/svc-headless.yaml | 0 .../kafka/charts/zookeeper/templates/svc.yaml | 0 .../kafka/charts/zookeeper/values-production.yaml | 0 .../charts/kafka/charts/zookeeper/values.yaml | 0 .../databases/charts/kafka/files/jks/README.md | 0 .../helmcharts/databases/charts/kafka/kafka.yaml | 0 .../databases/charts/kafka/requirements.lock | 0 .../databases/charts/kafka/requirements.yaml | 0 .../databases/charts/kafka/templates/NOTES.txt | 0 .../databases/charts/kafka/templates/_helpers.tpl | 0 .../databases/charts/kafka/templates/configmap.yaml | 0 .../charts/kafka/templates/extra-list.yaml | 0 .../charts/kafka/templates/jaas-secret.yaml | 0 .../charts/kafka/templates/jks-secret.yaml | 0 .../charts/kafka/templates/jmx-configmap.yaml | 0 .../charts/kafka/templates/jmx-metrics-svc.yaml | 0 .../kafka/templates/kafka-metrics-deployment.yaml | 0 .../charts/kafka/templates/kafka-metrics-svc.yaml | 0 .../charts/kafka/templates/log4j-configmap.yaml | 0 .../charts/kafka/templates/poddisruptionbudget.yaml | 0 .../databases/charts/kafka/templates/role.yaml | 0 .../charts/kafka/templates/rolebinding.yaml | 0 .../charts/kafka/templates/scripts-configmap.yaml | 0 .../charts/kafka/templates/serviceaccount.yaml | 0 .../kafka/templates/servicemonitor-jmx-metrics.yaml | 0 .../kafka/templates/servicemonitor-metrics.yaml | 0 .../charts/kafka/templates/statefulset.yaml | 0 .../charts/kafka/templates/svc-external-access.yaml | 0 .../charts/kafka/templates/svc-headless.yaml | 0 .../databases/charts/kafka/templates/svc.yaml | 0 .../databases/charts/kafka/values-production.yaml | 0 .../helmcharts/databases/charts/kafka/values.yaml | 0 .../helmcharts/databases/charts/minio/.helmignore | 0 .../helmcharts/databases/charts/minio/Chart.yaml | 0 .../helmcharts/databases/charts/minio/README.md | 0 .../charts/minio/ci/values-production.yaml | 0 .../databases/charts/minio/templates/NOTES.txt | 0 .../databases/charts/minio/templates/_helpers.tpl | 0 .../minio/templates/deployment-standalone.yaml | 0 .../databases/charts/minio/templates/ingress.yaml | 0 .../charts/minio/templates/networkpolicy.yaml | 0 .../charts/minio/templates/pvc-standalone.yaml | 0 .../databases/charts/minio/templates/secrets.yaml | 0 .../databases/charts/minio/templates/service.yaml | 0 .../charts/minio/templates/serviceaccount.yaml | 0 .../charts/minio/templates/statefulset.yaml | 0 .../charts/minio/templates/svc-headless.yaml | 0 .../charts/minio/templates/tls-secrets.yaml | 0 .../databases/charts/minio/values-production.yaml | 0 .../helmcharts/databases/charts/minio/values.yaml | 0 .../databases/charts/postgresql/.helmignore | 0 .../databases/charts/postgresql/Chart.yaml | 0 .../databases/charts/postgresql/README.md | 0 .../charts/postgresql/charts/common/.helmignore | 0 .../charts/postgresql/charts/common/Chart.yaml | 0 .../charts/postgresql/charts/common/README.md | 0 .../charts/common/templates/_affinities.tpl | 0 .../charts/common/templates/_capabilities.tpl | 0 .../postgresql/charts/common/templates/_errors.tpl | 0 .../postgresql/charts/common/templates/_images.tpl | 0 .../postgresql/charts/common/templates/_labels.tpl | 0 .../postgresql/charts/common/templates/_names.tpl | 0 .../postgresql/charts/common/templates/_secrets.tpl | 0 .../postgresql/charts/common/templates/_storage.tpl | 0 .../charts/common/templates/_tplvalues.tpl | 0 .../postgresql/charts/common/templates/_utils.tpl | 0 .../charts/common/templates/_validations.tpl | 0 .../charts/common/templates/_warnings.tpl | 0 .../charts/postgresql/charts/common/values.yaml | 0 .../charts/postgresql/ci/commonAnnotations.yaml | 0 .../charts/postgresql/ci/default-values.yaml | 0 .../postgresql/ci/shmvolume-disabled-values.yaml | 0 .../databases/charts/postgresql/files/README.md | 0 .../charts/postgresql/files/conf.d/README.md | 0 .../files/docker-entrypoint-initdb.d/README.md | 0 .../databases/charts/postgresql/postgresql.yaml | 0 .../databases/charts/postgresql/requirements.lock | 0 .../databases/charts/postgresql/requirements.yaml | 0 .../databases/charts/postgresql/templates/NOTES.txt | 0 .../charts/postgresql/templates/_helpers.tpl | 0 .../charts/postgresql/templates/configmap.yaml | 0 .../templates/extended-config-configmap.yaml | 0 .../charts/postgresql/templates/extra-list.yaml | 0 .../templates/initialization-configmap.yaml | 0 .../postgresql/templates/metrics-configmap.yaml | 0 .../charts/postgresql/templates/metrics-svc.yaml | 0 .../charts/postgresql/templates/networkpolicy.yaml | 0 .../postgresql/templates/podsecuritypolicy.yaml | 0 .../charts/postgresql/templates/prometheusrule.yaml | 0 .../databases/charts/postgresql/templates/role.yaml | 0 .../charts/postgresql/templates/rolebinding.yaml | 0 .../charts/postgresql/templates/secrets.yaml | 0 .../charts/postgresql/templates/serviceaccount.yaml | 0 .../charts/postgresql/templates/servicemonitor.yaml | 0 .../postgresql/templates/statefulset-slaves.yaml | 0 .../charts/postgresql/templates/statefulset.yaml | 0 .../charts/postgresql/templates/svc-headless.yaml | 0 .../charts/postgresql/templates/svc-read.yaml | 0 .../databases/charts/postgresql/templates/svc.yaml | 0 .../charts/postgresql/values-production.yaml | 0 .../databases/charts/postgresql/values.schema.json | 0 .../databases/charts/postgresql/values.yaml | 0 .../helmcharts/databases/charts/redis/.helmignore | 0 .../helmcharts/databases/charts/redis/Chart.lock | 0 .../helmcharts/databases/charts/redis/Chart.yaml | 0 .../helmcharts/databases/charts/redis/README.md | 0 .../charts/redis/charts/common/.helmignore | 0 .../databases/charts/redis/charts/common/Chart.yaml | 0 .../databases/charts/redis/charts/common/README.md | 0 .../redis/charts/common/templates/_affinities.tpl | 0 .../redis/charts/common/templates/_capabilities.tpl | 0 .../redis/charts/common/templates/_errors.tpl | 0 .../redis/charts/common/templates/_images.tpl | 0 .../redis/charts/common/templates/_ingress.tpl | 0 .../redis/charts/common/templates/_labels.tpl | 0 .../charts/redis/charts/common/templates/_names.tpl | 0 .../redis/charts/common/templates/_secrets.tpl | 0 .../redis/charts/common/templates/_storage.tpl | 0 .../redis/charts/common/templates/_tplvalues.tpl | 0 .../charts/redis/charts/common/templates/_utils.tpl | 0 .../redis/charts/common/templates/_warnings.tpl | 0 .../common/templates/validations/_cassandra.tpl | 0 .../common/templates/validations/_mariadb.tpl | 0 .../common/templates/validations/_mongodb.tpl | 0 .../common/templates/validations/_postgresql.tpl | 0 .../charts/common/templates/validations/_redis.tpl | 0 .../common/templates/validations/_validations.tpl | 0 .../charts/redis/charts/common/values.yaml | 0 .../databases/charts/redis/ci/default-values.yaml | 0 .../charts/redis/ci/extra-flags-values.yaml | 0 .../charts/redis/ci/production-sentinel-values.yaml | 0 .../charts/redis/img/redis-cluster-topology.png | Bin .../databases/charts/redis/img/redis-topology.png | Bin .../databases/charts/redis/templates/NOTES.txt | 0 .../databases/charts/redis/templates/_helpers.tpl | 0 .../charts/redis/templates/configmap-scripts.yaml | 0 .../databases/charts/redis/templates/configmap.yaml | 0 .../charts/redis/templates/headless-svc.yaml | 0 .../charts/redis/templates/health-configmap.yaml | 0 .../charts/redis/templates/metrics-prometheus.yaml | 0 .../templates/metrics-sentinel-prometheus.yaml | 0 .../redis/templates/metrics-sentinel-svc.yaml | 0 .../charts/redis/templates/metrics-svc.yaml | 0 .../charts/redis/templates/networkpolicy.yaml | 0 .../databases/charts/redis/templates/pdb.yaml | 0 .../charts/redis/templates/prometheusrule.yaml | 0 .../databases/charts/redis/templates/psp.yaml | 0 .../redis/templates/redis-master-statefulset.yaml | 0 .../charts/redis/templates/redis-master-svc.yaml | 0 .../redis/templates/redis-node-statefulset.yaml | 0 .../charts/redis/templates/redis-role.yaml | 0 .../charts/redis/templates/redis-rolebinding.yaml | 0 .../redis/templates/redis-serviceaccount.yaml | 0 .../redis/templates/redis-slave-statefulset.yaml | 0 .../charts/redis/templates/redis-slave-svc.yaml | 0 .../redis/templates/redis-with-sentinel-svc.yaml | 0 .../databases/charts/redis/templates/secret.yaml | 0 .../databases/charts/redis/values.schema.json | 0 .../helmcharts/databases/charts/redis/values.yaml | 0 .../helmcharts/databases/templates/NOTES.txt | 0 .../helmcharts/databases/templates/_helpers.tpl | 0 scripts/{helm => }/helmcharts/databases/values.yaml | 0 scripts/{helm => }/helmcharts/init.sh | 0 scripts/helmcharts/openreplay-cli | 1 + .../{helm => }/helmcharts/openreplay/.helmignore | 0 scripts/{helm => }/helmcharts/openreplay/Chart.yaml | 0 .../helmcharts/openreplay/charts/alerts/.helmignore | 0 .../helmcharts/openreplay/charts/alerts/Chart.yaml | 0 .../openreplay/charts/alerts/templates/NOTES.txt | 0 .../openreplay/charts/alerts/templates/_helpers.tpl | 0 .../charts/alerts/templates/deployment.yaml | 0 .../openreplay/charts/alerts/templates/hpa.yaml | 0 .../openreplay/charts/alerts/templates/ingress.yaml | 0 .../openreplay/charts/alerts/templates/service.yaml | 0 .../charts/alerts/templates/serviceaccount.yaml | 0 .../alerts/templates/tests/test-connection.yaml | 0 .../helmcharts/openreplay/charts/alerts/values.yaml | 0 .../helmcharts/openreplay/charts/assets/.helmignore | 0 .../helmcharts/openreplay/charts/assets/Chart.yaml | 0 .../openreplay/charts/assets/templates/NOTES.txt | 0 .../openreplay/charts/assets/templates/_helpers.tpl | 0 .../charts/assets/templates/deployment.yaml | 0 .../openreplay/charts/assets/templates/hpa.yaml | 0 .../openreplay/charts/assets/templates/ingress.yaml | 0 .../openreplay/charts/assets/templates/service.yaml | 0 .../charts/assets/templates/serviceaccount.yaml | 0 .../assets/templates/tests/test-connection.yaml | 0 .../helmcharts/openreplay/charts/assets/values.yaml | 0 .../openreplay/charts/chalice/.helmignore | 0 .../helmcharts/openreplay/charts/chalice/Chart.yaml | 0 .../openreplay/charts/chalice/templates/NOTES.txt | 0 .../charts/chalice/templates/_helpers.tpl | 0 .../charts/chalice/templates/deployment.yaml | 0 .../openreplay/charts/chalice/templates/hpa.yaml | 0 .../charts/chalice/templates/ingress.yaml | 0 .../charts/chalice/templates/service.yaml | 0 .../charts/chalice/templates/serviceaccount.yaml | 0 .../chalice/templates/tests/test-connection.yaml | 0 .../openreplay/charts/chalice/values.yaml | 0 .../helmcharts/openreplay/charts/db/.helmignore | 0 .../helmcharts/openreplay/charts/db/Chart.yaml | 0 .../openreplay/charts/db/templates/NOTES.txt | 0 .../openreplay/charts/db/templates/_helpers.tpl | 0 .../openreplay/charts/db/templates/deployment.yaml | 0 .../openreplay/charts/db/templates/hpa.yaml | 0 .../openreplay/charts/db/templates/ingress.yaml | 0 .../openreplay/charts/db/templates/service.yaml | 0 .../charts/db/templates/serviceaccount.yaml | 0 .../charts/db/templates/tests/test-connection.yaml | 0 .../helmcharts/openreplay/charts/db/values.yaml | 0 .../helmcharts/openreplay/charts/ender/.helmignore | 0 .../helmcharts/openreplay/charts/ender/Chart.yaml | 0 .../openreplay/charts/ender/templates/NOTES.txt | 0 .../openreplay/charts/ender/templates/_helpers.tpl | 0 .../charts/ender/templates/deployment.yaml | 0 .../openreplay/charts/ender/templates/hpa.yaml | 0 .../openreplay/charts/ender/templates/ingress.yaml | 0 .../openreplay/charts/ender/templates/service.yaml | 0 .../charts/ender/templates/serviceaccount.yaml | 0 .../ender/templates/tests/test-connection.yaml | 0 .../helmcharts/openreplay/charts/ender/values.yaml | 0 .../helmcharts/openreplay/charts/http/.helmignore | 0 .../helmcharts/openreplay/charts/http/Chart.yaml | 0 .../openreplay/charts/http/templates/NOTES.txt | 0 .../openreplay/charts/http/templates/_helpers.tpl | 0 .../charts/http/templates/deployment.yaml | 0 .../openreplay/charts/http/templates/hpa.yaml | 0 .../openreplay/charts/http/templates/ingress.yaml | 0 .../openreplay/charts/http/templates/service.yaml | 0 .../charts/http/templates/serviceaccount.yaml | 0 .../http/templates/tests/test-connection.yaml | 0 .../helmcharts/openreplay/charts/http/values.yaml | 0 .../openreplay/charts/integrations/.helmignore | 0 .../openreplay/charts/integrations/Chart.yaml | 0 .../charts/integrations/templates/NOTES.txt | 0 .../charts/integrations/templates/_helpers.tpl | 0 .../charts/integrations/templates/deployment.yaml | 0 .../charts/integrations/templates/hpa.yaml | 0 .../charts/integrations/templates/ingress.yaml | 0 .../charts/integrations/templates/service.yaml | 0 .../integrations/templates/serviceaccount.yaml | 0 .../templates/tests/test-connection.yaml | 0 .../openreplay/charts/integrations/values.yaml | 0 .../openreplay/charts/nginx-ingress/.helmignore | 0 .../openreplay/charts/nginx-ingress/Chart.yaml | 0 .../charts/nginx-ingress/templates/NOTES.txt | 0 .../charts/nginx-ingress/templates/_helpers.tpl | 0 .../charts/nginx-ingress/templates/configMap.yaml | 0 .../charts/nginx-ingress/templates/deployment.yaml | 0 .../charts/nginx-ingress/templates/hpa.yaml | 0 .../charts/nginx-ingress/templates/ingress.yaml | 0 .../charts/nginx-ingress/templates/secrets.yaml | 0 .../charts/nginx-ingress/templates/service.yaml | 0 .../nginx-ingress/templates/serviceaccount.yaml | 0 .../templates/tests/test-connection.yaml | 0 .../openreplay/charts/nginx-ingress/values.yaml | 0 .../helmcharts/openreplay/charts/sink/.helmignore | 0 .../helmcharts/openreplay/charts/sink/Chart.yaml | 0 .../openreplay/charts/sink/templates/NOTES.txt | 0 .../openreplay/charts/sink/templates/_helpers.tpl | 0 .../charts/sink/templates/deployment.yaml | 0 .../openreplay/charts/sink/templates/hpa.yaml | 0 .../openreplay/charts/sink/templates/ingress.yaml | 0 .../openreplay/charts/sink/templates/service.yaml | 0 .../charts/sink/templates/serviceaccount.yaml | 0 .../sink/templates/tests/test-connection.yaml | 0 .../helmcharts/openreplay/charts/sink/values.yaml | 0 .../openreplay/charts/storage/.helmignore | 0 .../helmcharts/openreplay/charts/storage/Chart.yaml | 0 .../openreplay/charts/storage/templates/NOTES.txt | 0 .../charts/storage/templates/_helpers.tpl | 0 .../charts/storage/templates/deployment.yaml | 0 .../openreplay/charts/storage/templates/hpa.yaml | 0 .../charts/storage/templates/ingress.yaml | 0 .../charts/storage/templates/service.yaml | 0 .../charts/storage/templates/serviceaccount.yaml | 0 .../storage/templates/tests/test-connection.yaml | 0 .../openreplay/charts/storage/values.yaml | 0 .../openreplay/charts/utilities/.helmignore | 0 .../openreplay/charts/utilities/Chart.yaml | 0 .../openreplay/charts/utilities/templates/NOTES.txt | 0 .../charts/utilities/templates/_helpers.tpl | 0 .../charts/utilities/templates/deployment.yaml | 0 .../openreplay/charts/utilities/templates/hpa.yaml | 0 .../charts/utilities/templates/ingress.yaml | 0 .../charts/utilities/templates/service.yaml | 0 .../charts/utilities/templates/serviceaccount.yaml | 0 .../utilities/templates/tests/test-connection.yaml | 0 .../openreplay/charts/utilities/values.yaml | 0 .../helmcharts/openreplay/files/clickhouse.sh | 0 .../{helm => }/helmcharts/openreplay/files/dbops.sh | 0 .../{helm => }/helmcharts/openreplay/files/kafka.sh | 0 .../{helm => }/helmcharts/openreplay/files/minio.sh | 0 .../helmcharts/openreplay/files/postgresql.sh | 0 .../helmcharts/openreplay/templates/NOTES.txt | 0 .../helmcharts/openreplay/templates/_helpers.tpl | 0 .../helmcharts/openreplay/templates/job.yaml | 0 .../openreplay/templates/serviceaccount.yaml | 0 .../{helm => }/helmcharts/openreplay/values.yaml | 0 scripts/{helm => }/helmcharts/vars.yaml | 0 scripts/{helm => }/helmcharts/versionUpdater.sh | 0 329 files changed, 2 insertions(+), 1 deletion(-) rename scripts/{helm => }/helmcharts/README.md (100%) rename scripts/{helm => }/helmcharts/databases/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/clickhouse/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/files/jks/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/kafka.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/requirements.lock (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/requirements.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/extra-list.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/jks-secret.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/role.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/rolebinding.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/svc-headless.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/templates/svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/values-production.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/kafka/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/ci/values-production.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/networkpolicy.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/secrets.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/svc-headless.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/templates/tls-secrets.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/values-production.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/minio/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/charts/common/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/ci/default-values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/files/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/files/conf.d/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/postgresql.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/requirements.lock (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/requirements.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/extra-list.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/role.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/secrets.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/svc-read.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/templates/svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/values-production.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/values.schema.json (100%) rename scripts/{helm => }/helmcharts/databases/charts/postgresql/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/Chart.lock (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/.helmignore (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/README.md (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/charts/common/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/ci/default-values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/img/redis-cluster-topology.png (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/img/redis-topology.png (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/headless-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/health-configmap.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/metrics-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/networkpolicy.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/pdb.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/prometheusrule.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/psp.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-role.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/templates/secret.yaml (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/values.schema.json (100%) rename scripts/{helm => }/helmcharts/databases/charts/redis/values.yaml (100%) rename scripts/{helm => }/helmcharts/databases/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/databases/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/databases/values.yaml (100%) rename scripts/{helm => }/helmcharts/init.sh (100%) create mode 120000 scripts/helmcharts/openreplay-cli rename scripts/{helm => }/helmcharts/openreplay/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/alerts/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/assets/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/chalice/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/db/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/ender/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/http/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/integrations/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/nginx-ingress/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/sink/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/storage/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/.helmignore (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/Chart.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/deployment.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/hpa.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/ingress.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/service.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/charts/utilities/values.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/files/clickhouse.sh (100%) rename scripts/{helm => }/helmcharts/openreplay/files/dbops.sh (100%) rename scripts/{helm => }/helmcharts/openreplay/files/kafka.sh (100%) rename scripts/{helm => }/helmcharts/openreplay/files/minio.sh (100%) rename scripts/{helm => }/helmcharts/openreplay/files/postgresql.sh (100%) rename scripts/{helm => }/helmcharts/openreplay/templates/NOTES.txt (100%) rename scripts/{helm => }/helmcharts/openreplay/templates/_helpers.tpl (100%) rename scripts/{helm => }/helmcharts/openreplay/templates/job.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/templates/serviceaccount.yaml (100%) rename scripts/{helm => }/helmcharts/openreplay/values.yaml (100%) rename scripts/{helm => }/helmcharts/vars.yaml (100%) rename scripts/{helm => }/helmcharts/versionUpdater.sh (100%) diff --git a/scripts/helm/openreplay-cli b/scripts/helm/openreplay-cli index 0f229e93a..6fa9472f6 100755 --- a/scripts/helm/openreplay-cli +++ b/scripts/helm/openreplay-cli @@ -109,7 +109,7 @@ legacyInstall() { } helmInstall() { - helm upgrade --install openreplay helmcharts/openreplay -f helmcharts/vars.yaml + helm upgrade --install openreplay openreplay -f vars.yaml } upgrade() { diff --git a/scripts/helm/helmcharts/README.md b/scripts/helmcharts/README.md similarity index 100% rename from scripts/helm/helmcharts/README.md rename to scripts/helmcharts/README.md diff --git a/scripts/helm/helmcharts/databases/.helmignore b/scripts/helmcharts/databases/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/.helmignore rename to scripts/helmcharts/databases/.helmignore diff --git a/scripts/helm/helmcharts/databases/Chart.yaml b/scripts/helmcharts/databases/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/Chart.yaml rename to scripts/helmcharts/databases/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/.helmignore b/scripts/helmcharts/databases/charts/clickhouse/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/.helmignore rename to scripts/helmcharts/databases/charts/clickhouse/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/Chart.yaml b/scripts/helmcharts/databases/charts/clickhouse/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/Chart.yaml rename to scripts/helmcharts/databases/charts/clickhouse/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl rename to scripts/helmcharts/databases/charts/clickhouse/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/templates/service.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/templates/service.yaml rename to scripts/helmcharts/databases/charts/clickhouse/templates/service.yaml diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml rename to scripts/helmcharts/databases/charts/clickhouse/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml b/scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml rename to scripts/helmcharts/databases/charts/clickhouse/templates/statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/clickhouse/values.yaml b/scripts/helmcharts/databases/charts/clickhouse/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/clickhouse/values.yaml rename to scripts/helmcharts/databases/charts/clickhouse/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/.helmignore b/scripts/helmcharts/databases/charts/kafka/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/.helmignore rename to scripts/helmcharts/databases/charts/kafka/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/kafka/Chart.yaml b/scripts/helmcharts/databases/charts/kafka/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/Chart.yaml rename to scripts/helmcharts/databases/charts/kafka/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/README.md b/scripts/helmcharts/databases/charts/kafka/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/README.md rename to scripts/helmcharts/databases/charts/kafka/README.md diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/README.md b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/README.md rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/README.md diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/metrics-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/networkpolicy.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/poddisruptionbudget.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/prometheusrules.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/secrets.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/servicemonitor.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc-headless.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/templates/svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/values-production.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml b/scripts/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml rename to scripts/helmcharts/databases/charts/kafka/charts/zookeeper/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/files/jks/README.md b/scripts/helmcharts/databases/charts/kafka/files/jks/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/files/jks/README.md rename to scripts/helmcharts/databases/charts/kafka/files/jks/README.md diff --git a/scripts/helm/helmcharts/databases/charts/kafka/kafka.yaml b/scripts/helmcharts/databases/charts/kafka/kafka.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/kafka.yaml rename to scripts/helmcharts/databases/charts/kafka/kafka.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/requirements.lock b/scripts/helmcharts/databases/charts/kafka/requirements.lock similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/requirements.lock rename to scripts/helmcharts/databases/charts/kafka/requirements.lock diff --git a/scripts/helm/helmcharts/databases/charts/kafka/requirements.yaml b/scripts/helmcharts/databases/charts/kafka/requirements.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/requirements.yaml rename to scripts/helmcharts/databases/charts/kafka/requirements.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/NOTES.txt b/scripts/helmcharts/databases/charts/kafka/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/NOTES.txt rename to scripts/helmcharts/databases/charts/kafka/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/kafka/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/_helpers.tpl rename to scripts/helmcharts/databases/charts/kafka/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/configmap.yaml b/scripts/helmcharts/databases/charts/kafka/templates/configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/configmap.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/extra-list.yaml b/scripts/helmcharts/databases/charts/kafka/templates/extra-list.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/extra-list.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/extra-list.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml b/scripts/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/jaas-secret.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/jks-secret.yaml b/scripts/helmcharts/databases/charts/kafka/templates/jks-secret.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/jks-secret.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/jks-secret.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml b/scripts/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/jmx-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml b/scripts/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/jmx-metrics-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml b/scripts/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/kafka-metrics-deployment.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml b/scripts/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/kafka-metrics-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml b/scripts/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/log4j-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml b/scripts/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/poddisruptionbudget.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/role.yaml b/scripts/helmcharts/databases/charts/kafka/templates/role.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/role.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/role.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/rolebinding.yaml b/scripts/helmcharts/databases/charts/kafka/templates/rolebinding.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/rolebinding.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/rolebinding.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml b/scripts/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/scripts-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml b/scripts/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml b/scripts/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/servicemonitor-jmx-metrics.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml b/scripts/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/servicemonitor-metrics.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/statefulset.yaml b/scripts/helmcharts/databases/charts/kafka/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/statefulset.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml b/scripts/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/svc-external-access.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/svc-headless.yaml b/scripts/helmcharts/databases/charts/kafka/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/svc-headless.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/svc-headless.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/templates/svc.yaml b/scripts/helmcharts/databases/charts/kafka/templates/svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/templates/svc.yaml rename to scripts/helmcharts/databases/charts/kafka/templates/svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/values-production.yaml b/scripts/helmcharts/databases/charts/kafka/values-production.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/values-production.yaml rename to scripts/helmcharts/databases/charts/kafka/values-production.yaml diff --git a/scripts/helm/helmcharts/databases/charts/kafka/values.yaml b/scripts/helmcharts/databases/charts/kafka/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/kafka/values.yaml rename to scripts/helmcharts/databases/charts/kafka/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/.helmignore b/scripts/helmcharts/databases/charts/minio/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/.helmignore rename to scripts/helmcharts/databases/charts/minio/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/minio/Chart.yaml b/scripts/helmcharts/databases/charts/minio/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/Chart.yaml rename to scripts/helmcharts/databases/charts/minio/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/README.md b/scripts/helmcharts/databases/charts/minio/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/README.md rename to scripts/helmcharts/databases/charts/minio/README.md diff --git a/scripts/helm/helmcharts/databases/charts/minio/ci/values-production.yaml b/scripts/helmcharts/databases/charts/minio/ci/values-production.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/ci/values-production.yaml rename to scripts/helmcharts/databases/charts/minio/ci/values-production.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/NOTES.txt b/scripts/helmcharts/databases/charts/minio/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/NOTES.txt rename to scripts/helmcharts/databases/charts/minio/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/minio/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/_helpers.tpl rename to scripts/helmcharts/databases/charts/minio/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml b/scripts/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml rename to scripts/helmcharts/databases/charts/minio/templates/deployment-standalone.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/ingress.yaml b/scripts/helmcharts/databases/charts/minio/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/ingress.yaml rename to scripts/helmcharts/databases/charts/minio/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/networkpolicy.yaml b/scripts/helmcharts/databases/charts/minio/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/networkpolicy.yaml rename to scripts/helmcharts/databases/charts/minio/templates/networkpolicy.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml b/scripts/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml rename to scripts/helmcharts/databases/charts/minio/templates/pvc-standalone.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/secrets.yaml b/scripts/helmcharts/databases/charts/minio/templates/secrets.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/secrets.yaml rename to scripts/helmcharts/databases/charts/minio/templates/secrets.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/service.yaml b/scripts/helmcharts/databases/charts/minio/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/service.yaml rename to scripts/helmcharts/databases/charts/minio/templates/service.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/serviceaccount.yaml b/scripts/helmcharts/databases/charts/minio/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/serviceaccount.yaml rename to scripts/helmcharts/databases/charts/minio/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/statefulset.yaml b/scripts/helmcharts/databases/charts/minio/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/statefulset.yaml rename to scripts/helmcharts/databases/charts/minio/templates/statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/svc-headless.yaml b/scripts/helmcharts/databases/charts/minio/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/svc-headless.yaml rename to scripts/helmcharts/databases/charts/minio/templates/svc-headless.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/templates/tls-secrets.yaml b/scripts/helmcharts/databases/charts/minio/templates/tls-secrets.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/templates/tls-secrets.yaml rename to scripts/helmcharts/databases/charts/minio/templates/tls-secrets.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/values-production.yaml b/scripts/helmcharts/databases/charts/minio/values-production.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/values-production.yaml rename to scripts/helmcharts/databases/charts/minio/values-production.yaml diff --git a/scripts/helm/helmcharts/databases/charts/minio/values.yaml b/scripts/helmcharts/databases/charts/minio/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/minio/values.yaml rename to scripts/helmcharts/databases/charts/minio/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/.helmignore b/scripts/helmcharts/databases/charts/postgresql/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/.helmignore rename to scripts/helmcharts/databases/charts/postgresql/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/Chart.yaml b/scripts/helmcharts/databases/charts/postgresql/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/Chart.yaml rename to scripts/helmcharts/databases/charts/postgresql/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/README.md b/scripts/helmcharts/databases/charts/postgresql/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/README.md rename to scripts/helmcharts/databases/charts/postgresql/README.md diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/.helmignore b/scripts/helmcharts/databases/charts/postgresql/charts/common/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/.helmignore rename to scripts/helmcharts/databases/charts/postgresql/charts/common/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml b/scripts/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml rename to scripts/helmcharts/databases/charts/postgresql/charts/common/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/README.md b/scripts/helmcharts/databases/charts/postgresql/charts/common/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/README.md rename to scripts/helmcharts/databases/charts/postgresql/charts/common/README.md diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_affinities.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_capabilities.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_errors.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_images.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_labels.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_names.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_secrets.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_storage.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_tplvalues.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_utils.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_validations.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl b/scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl rename to scripts/helmcharts/databases/charts/postgresql/charts/common/templates/_warnings.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/charts/common/values.yaml b/scripts/helmcharts/databases/charts/postgresql/charts/common/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/charts/common/values.yaml rename to scripts/helmcharts/databases/charts/postgresql/charts/common/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml b/scripts/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml rename to scripts/helmcharts/databases/charts/postgresql/ci/commonAnnotations.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/ci/default-values.yaml b/scripts/helmcharts/databases/charts/postgresql/ci/default-values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/ci/default-values.yaml rename to scripts/helmcharts/databases/charts/postgresql/ci/default-values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml b/scripts/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml rename to scripts/helmcharts/databases/charts/postgresql/ci/shmvolume-disabled-values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/files/README.md b/scripts/helmcharts/databases/charts/postgresql/files/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/files/README.md rename to scripts/helmcharts/databases/charts/postgresql/files/README.md diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/files/conf.d/README.md b/scripts/helmcharts/databases/charts/postgresql/files/conf.d/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/files/conf.d/README.md rename to scripts/helmcharts/databases/charts/postgresql/files/conf.d/README.md diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md b/scripts/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md rename to scripts/helmcharts/databases/charts/postgresql/files/docker-entrypoint-initdb.d/README.md diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/postgresql.yaml b/scripts/helmcharts/databases/charts/postgresql/postgresql.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/postgresql.yaml rename to scripts/helmcharts/databases/charts/postgresql/postgresql.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/requirements.lock b/scripts/helmcharts/databases/charts/postgresql/requirements.lock similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/requirements.lock rename to scripts/helmcharts/databases/charts/postgresql/requirements.lock diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/requirements.yaml b/scripts/helmcharts/databases/charts/postgresql/requirements.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/requirements.yaml rename to scripts/helmcharts/databases/charts/postgresql/requirements.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/NOTES.txt b/scripts/helmcharts/databases/charts/postgresql/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/NOTES.txt rename to scripts/helmcharts/databases/charts/postgresql/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/postgresql/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/_helpers.tpl rename to scripts/helmcharts/databases/charts/postgresql/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/configmap.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/configmap.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/extended-config-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/extra-list.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/extra-list.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/extra-list.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/extra-list.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/initialization-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/metrics-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/metrics-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/networkpolicy.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/podsecuritypolicy.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/prometheusrule.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/role.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/role.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/role.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/role.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/rolebinding.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/secrets.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/secrets.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/secrets.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/secrets.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/servicemonitor.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/statefulset-slaves.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/statefulset.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/svc-headless.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-read.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/svc-read.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/svc-read.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/svc-read.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/templates/svc.yaml b/scripts/helmcharts/databases/charts/postgresql/templates/svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/templates/svc.yaml rename to scripts/helmcharts/databases/charts/postgresql/templates/svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/values-production.yaml b/scripts/helmcharts/databases/charts/postgresql/values-production.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/values-production.yaml rename to scripts/helmcharts/databases/charts/postgresql/values-production.yaml diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/values.schema.json b/scripts/helmcharts/databases/charts/postgresql/values.schema.json similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/values.schema.json rename to scripts/helmcharts/databases/charts/postgresql/values.schema.json diff --git a/scripts/helm/helmcharts/databases/charts/postgresql/values.yaml b/scripts/helmcharts/databases/charts/postgresql/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/postgresql/values.yaml rename to scripts/helmcharts/databases/charts/postgresql/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/.helmignore b/scripts/helmcharts/databases/charts/redis/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/.helmignore rename to scripts/helmcharts/databases/charts/redis/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/redis/Chart.lock b/scripts/helmcharts/databases/charts/redis/Chart.lock similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/Chart.lock rename to scripts/helmcharts/databases/charts/redis/Chart.lock diff --git a/scripts/helm/helmcharts/databases/charts/redis/Chart.yaml b/scripts/helmcharts/databases/charts/redis/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/Chart.yaml rename to scripts/helmcharts/databases/charts/redis/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/README.md b/scripts/helmcharts/databases/charts/redis/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/README.md rename to scripts/helmcharts/databases/charts/redis/README.md diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/.helmignore b/scripts/helmcharts/databases/charts/redis/charts/common/.helmignore similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/.helmignore rename to scripts/helmcharts/databases/charts/redis/charts/common/.helmignore diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/Chart.yaml b/scripts/helmcharts/databases/charts/redis/charts/common/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/Chart.yaml rename to scripts/helmcharts/databases/charts/redis/charts/common/Chart.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/README.md b/scripts/helmcharts/databases/charts/redis/charts/common/README.md similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/README.md rename to scripts/helmcharts/databases/charts/redis/charts/common/README.md diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_affinities.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_capabilities.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_errors.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_images.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_ingress.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_labels.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_names.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_secrets.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_storage.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_tplvalues.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_utils.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/_warnings.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_cassandra.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_mariadb.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_mongodb.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_postgresql.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_redis.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl b/scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl rename to scripts/helmcharts/databases/charts/redis/charts/common/templates/validations/_validations.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/charts/common/values.yaml b/scripts/helmcharts/databases/charts/redis/charts/common/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/charts/common/values.yaml rename to scripts/helmcharts/databases/charts/redis/charts/common/values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/ci/default-values.yaml b/scripts/helmcharts/databases/charts/redis/ci/default-values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/ci/default-values.yaml rename to scripts/helmcharts/databases/charts/redis/ci/default-values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml b/scripts/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml rename to scripts/helmcharts/databases/charts/redis/ci/extra-flags-values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml b/scripts/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml rename to scripts/helmcharts/databases/charts/redis/ci/production-sentinel-values.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/img/redis-cluster-topology.png b/scripts/helmcharts/databases/charts/redis/img/redis-cluster-topology.png similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/img/redis-cluster-topology.png rename to scripts/helmcharts/databases/charts/redis/img/redis-cluster-topology.png diff --git a/scripts/helm/helmcharts/databases/charts/redis/img/redis-topology.png b/scripts/helmcharts/databases/charts/redis/img/redis-topology.png similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/img/redis-topology.png rename to scripts/helmcharts/databases/charts/redis/img/redis-topology.png diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/NOTES.txt b/scripts/helmcharts/databases/charts/redis/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/NOTES.txt rename to scripts/helmcharts/databases/charts/redis/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/_helpers.tpl b/scripts/helmcharts/databases/charts/redis/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/_helpers.tpl rename to scripts/helmcharts/databases/charts/redis/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml b/scripts/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml rename to scripts/helmcharts/databases/charts/redis/templates/configmap-scripts.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/configmap.yaml b/scripts/helmcharts/databases/charts/redis/templates/configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/configmap.yaml rename to scripts/helmcharts/databases/charts/redis/templates/configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/headless-svc.yaml b/scripts/helmcharts/databases/charts/redis/templates/headless-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/headless-svc.yaml rename to scripts/helmcharts/databases/charts/redis/templates/headless-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/health-configmap.yaml b/scripts/helmcharts/databases/charts/redis/templates/health-configmap.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/health-configmap.yaml rename to scripts/helmcharts/databases/charts/redis/templates/health-configmap.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml b/scripts/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml rename to scripts/helmcharts/databases/charts/redis/templates/metrics-prometheus.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml b/scripts/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml rename to scripts/helmcharts/databases/charts/redis/templates/metrics-sentinel-prometheus.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml b/scripts/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml rename to scripts/helmcharts/databases/charts/redis/templates/metrics-sentinel-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/metrics-svc.yaml b/scripts/helmcharts/databases/charts/redis/templates/metrics-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/metrics-svc.yaml rename to scripts/helmcharts/databases/charts/redis/templates/metrics-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/networkpolicy.yaml b/scripts/helmcharts/databases/charts/redis/templates/networkpolicy.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/networkpolicy.yaml rename to scripts/helmcharts/databases/charts/redis/templates/networkpolicy.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/pdb.yaml b/scripts/helmcharts/databases/charts/redis/templates/pdb.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/pdb.yaml rename to scripts/helmcharts/databases/charts/redis/templates/pdb.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/prometheusrule.yaml b/scripts/helmcharts/databases/charts/redis/templates/prometheusrule.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/prometheusrule.yaml rename to scripts/helmcharts/databases/charts/redis/templates/prometheusrule.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/psp.yaml b/scripts/helmcharts/databases/charts/redis/templates/psp.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/psp.yaml rename to scripts/helmcharts/databases/charts/redis/templates/psp.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-master-statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-master-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-node-statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-role.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-role.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-role.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-role.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-rolebinding.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-serviceaccount.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-slave-statefulset.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-slave-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml b/scripts/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml rename to scripts/helmcharts/databases/charts/redis/templates/redis-with-sentinel-svc.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/templates/secret.yaml b/scripts/helmcharts/databases/charts/redis/templates/secret.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/templates/secret.yaml rename to scripts/helmcharts/databases/charts/redis/templates/secret.yaml diff --git a/scripts/helm/helmcharts/databases/charts/redis/values.schema.json b/scripts/helmcharts/databases/charts/redis/values.schema.json similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/values.schema.json rename to scripts/helmcharts/databases/charts/redis/values.schema.json diff --git a/scripts/helm/helmcharts/databases/charts/redis/values.yaml b/scripts/helmcharts/databases/charts/redis/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/charts/redis/values.yaml rename to scripts/helmcharts/databases/charts/redis/values.yaml diff --git a/scripts/helm/helmcharts/databases/templates/NOTES.txt b/scripts/helmcharts/databases/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/databases/templates/NOTES.txt rename to scripts/helmcharts/databases/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/databases/templates/_helpers.tpl b/scripts/helmcharts/databases/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/databases/templates/_helpers.tpl rename to scripts/helmcharts/databases/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/databases/values.yaml b/scripts/helmcharts/databases/values.yaml similarity index 100% rename from scripts/helm/helmcharts/databases/values.yaml rename to scripts/helmcharts/databases/values.yaml diff --git a/scripts/helm/helmcharts/init.sh b/scripts/helmcharts/init.sh similarity index 100% rename from scripts/helm/helmcharts/init.sh rename to scripts/helmcharts/init.sh diff --git a/scripts/helmcharts/openreplay-cli b/scripts/helmcharts/openreplay-cli new file mode 120000 index 000000000..d299dbb4f --- /dev/null +++ b/scripts/helmcharts/openreplay-cli @@ -0,0 +1 @@ +../helm/openreplay-cli \ No newline at end of file diff --git a/scripts/helm/helmcharts/openreplay/.helmignore b/scripts/helmcharts/openreplay/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/.helmignore rename to scripts/helmcharts/openreplay/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/Chart.yaml b/scripts/helmcharts/openreplay/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/Chart.yaml rename to scripts/helmcharts/openreplay/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/.helmignore b/scripts/helmcharts/openreplay/charts/alerts/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/.helmignore rename to scripts/helmcharts/openreplay/charts/alerts/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml b/scripts/helmcharts/openreplay/charts/alerts/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/Chart.yaml rename to scripts/helmcharts/openreplay/charts/alerts/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/alerts/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/alerts/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/alerts/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/alerts/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/alerts/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/service.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/alerts/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/alerts/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/alerts/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/alerts/values.yaml rename to scripts/helmcharts/openreplay/charts/alerts/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/.helmignore b/scripts/helmcharts/openreplay/charts/assets/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/.helmignore rename to scripts/helmcharts/openreplay/charts/assets/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml b/scripts/helmcharts/openreplay/charts/assets/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/Chart.yaml rename to scripts/helmcharts/openreplay/charts/assets/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/assets/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/assets/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/assets/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/assets/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/assets/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/assets/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/service.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/assets/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/assets/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/assets/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/assets/values.yaml b/scripts/helmcharts/openreplay/charts/assets/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/assets/values.yaml rename to scripts/helmcharts/openreplay/charts/assets/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/.helmignore b/scripts/helmcharts/openreplay/charts/chalice/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/.helmignore rename to scripts/helmcharts/openreplay/charts/chalice/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml b/scripts/helmcharts/openreplay/charts/chalice/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/Chart.yaml rename to scripts/helmcharts/openreplay/charts/chalice/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/chalice/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/chalice/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/chalice/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/chalice/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/chalice/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/service.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/chalice/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/chalice/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/chalice/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/chalice/values.yaml rename to scripts/helmcharts/openreplay/charts/chalice/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/.helmignore b/scripts/helmcharts/openreplay/charts/db/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/.helmignore rename to scripts/helmcharts/openreplay/charts/db/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml b/scripts/helmcharts/openreplay/charts/db/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/Chart.yaml rename to scripts/helmcharts/openreplay/charts/db/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/db/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/db/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/db/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/db/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/db/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/db/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/db/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/db/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/db/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/service.yaml b/scripts/helmcharts/openreplay/charts/db/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/db/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/db/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/db/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/db/values.yaml b/scripts/helmcharts/openreplay/charts/db/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/db/values.yaml rename to scripts/helmcharts/openreplay/charts/db/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/.helmignore b/scripts/helmcharts/openreplay/charts/ender/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/.helmignore rename to scripts/helmcharts/openreplay/charts/ender/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml b/scripts/helmcharts/openreplay/charts/ender/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/Chart.yaml rename to scripts/helmcharts/openreplay/charts/ender/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/ender/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/ender/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/ender/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/ender/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/ender/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/ender/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/ender/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/service.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/ender/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/ender/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/ender/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/ender/values.yaml b/scripts/helmcharts/openreplay/charts/ender/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/ender/values.yaml rename to scripts/helmcharts/openreplay/charts/ender/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/.helmignore b/scripts/helmcharts/openreplay/charts/http/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/.helmignore rename to scripts/helmcharts/openreplay/charts/http/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml b/scripts/helmcharts/openreplay/charts/http/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/Chart.yaml rename to scripts/helmcharts/openreplay/charts/http/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/http/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/http/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/http/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/http/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/http/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/http/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/http/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/service.yaml b/scripts/helmcharts/openreplay/charts/http/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/http/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/http/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/http/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/http/values.yaml b/scripts/helmcharts/openreplay/charts/http/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/http/values.yaml rename to scripts/helmcharts/openreplay/charts/http/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/.helmignore b/scripts/helmcharts/openreplay/charts/integrations/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/.helmignore rename to scripts/helmcharts/openreplay/charts/integrations/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml b/scripts/helmcharts/openreplay/charts/integrations/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/Chart.yaml rename to scripts/helmcharts/openreplay/charts/integrations/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/integrations/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/integrations/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/integrations/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/integrations/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/integrations/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/integrations/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/service.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/integrations/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/integrations/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/integrations/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/integrations/values.yaml b/scripts/helmcharts/openreplay/charts/integrations/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/integrations/values.yaml rename to scripts/helmcharts/openreplay/charts/integrations/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/.helmignore b/scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/.helmignore rename to scripts/helmcharts/openreplay/charts/nginx-ingress/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/nginx-ingress/values.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/nginx-ingress/values.yaml rename to scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/.helmignore b/scripts/helmcharts/openreplay/charts/sink/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/.helmignore rename to scripts/helmcharts/openreplay/charts/sink/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml b/scripts/helmcharts/openreplay/charts/sink/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/Chart.yaml rename to scripts/helmcharts/openreplay/charts/sink/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/sink/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/sink/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/sink/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/sink/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/sink/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/sink/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/sink/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/service.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/sink/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/sink/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/sink/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/sink/values.yaml b/scripts/helmcharts/openreplay/charts/sink/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/sink/values.yaml rename to scripts/helmcharts/openreplay/charts/sink/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/.helmignore b/scripts/helmcharts/openreplay/charts/storage/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/.helmignore rename to scripts/helmcharts/openreplay/charts/storage/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml b/scripts/helmcharts/openreplay/charts/storage/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/Chart.yaml rename to scripts/helmcharts/openreplay/charts/storage/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/storage/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/storage/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/storage/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/storage/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/storage/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/storage/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/service.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/storage/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/storage/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/storage/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/storage/values.yaml b/scripts/helmcharts/openreplay/charts/storage/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/storage/values.yaml rename to scripts/helmcharts/openreplay/charts/storage/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/.helmignore b/scripts/helmcharts/openreplay/charts/utilities/.helmignore similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/.helmignore rename to scripts/helmcharts/openreplay/charts/utilities/.helmignore diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml b/scripts/helmcharts/openreplay/charts/utilities/Chart.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/Chart.yaml rename to scripts/helmcharts/openreplay/charts/utilities/Chart.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/NOTES.txt b/scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/NOTES.txt rename to scripts/helmcharts/openreplay/charts/utilities/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl b/scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/charts/utilities/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/deployment.yaml rename to scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/hpa.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/hpa.yaml rename to scripts/helmcharts/openreplay/charts/utilities/templates/hpa.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/ingress.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/ingress.yaml rename to scripts/helmcharts/openreplay/charts/utilities/templates/ingress.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/service.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/service.yaml rename to scripts/helmcharts/openreplay/charts/utilities/templates/service.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/charts/utilities/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml rename to scripts/helmcharts/openreplay/charts/utilities/templates/tests/test-connection.yaml diff --git a/scripts/helm/helmcharts/openreplay/charts/utilities/values.yaml b/scripts/helmcharts/openreplay/charts/utilities/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/charts/utilities/values.yaml rename to scripts/helmcharts/openreplay/charts/utilities/values.yaml diff --git a/scripts/helm/helmcharts/openreplay/files/clickhouse.sh b/scripts/helmcharts/openreplay/files/clickhouse.sh similarity index 100% rename from scripts/helm/helmcharts/openreplay/files/clickhouse.sh rename to scripts/helmcharts/openreplay/files/clickhouse.sh diff --git a/scripts/helm/helmcharts/openreplay/files/dbops.sh b/scripts/helmcharts/openreplay/files/dbops.sh similarity index 100% rename from scripts/helm/helmcharts/openreplay/files/dbops.sh rename to scripts/helmcharts/openreplay/files/dbops.sh diff --git a/scripts/helm/helmcharts/openreplay/files/kafka.sh b/scripts/helmcharts/openreplay/files/kafka.sh similarity index 100% rename from scripts/helm/helmcharts/openreplay/files/kafka.sh rename to scripts/helmcharts/openreplay/files/kafka.sh diff --git a/scripts/helm/helmcharts/openreplay/files/minio.sh b/scripts/helmcharts/openreplay/files/minio.sh similarity index 100% rename from scripts/helm/helmcharts/openreplay/files/minio.sh rename to scripts/helmcharts/openreplay/files/minio.sh diff --git a/scripts/helm/helmcharts/openreplay/files/postgresql.sh b/scripts/helmcharts/openreplay/files/postgresql.sh similarity index 100% rename from scripts/helm/helmcharts/openreplay/files/postgresql.sh rename to scripts/helmcharts/openreplay/files/postgresql.sh diff --git a/scripts/helm/helmcharts/openreplay/templates/NOTES.txt b/scripts/helmcharts/openreplay/templates/NOTES.txt similarity index 100% rename from scripts/helm/helmcharts/openreplay/templates/NOTES.txt rename to scripts/helmcharts/openreplay/templates/NOTES.txt diff --git a/scripts/helm/helmcharts/openreplay/templates/_helpers.tpl b/scripts/helmcharts/openreplay/templates/_helpers.tpl similarity index 100% rename from scripts/helm/helmcharts/openreplay/templates/_helpers.tpl rename to scripts/helmcharts/openreplay/templates/_helpers.tpl diff --git a/scripts/helm/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/templates/job.yaml rename to scripts/helmcharts/openreplay/templates/job.yaml diff --git a/scripts/helm/helmcharts/openreplay/templates/serviceaccount.yaml b/scripts/helmcharts/openreplay/templates/serviceaccount.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/templates/serviceaccount.yaml rename to scripts/helmcharts/openreplay/templates/serviceaccount.yaml diff --git a/scripts/helm/helmcharts/openreplay/values.yaml b/scripts/helmcharts/openreplay/values.yaml similarity index 100% rename from scripts/helm/helmcharts/openreplay/values.yaml rename to scripts/helmcharts/openreplay/values.yaml diff --git a/scripts/helm/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml similarity index 100% rename from scripts/helm/helmcharts/vars.yaml rename to scripts/helmcharts/vars.yaml diff --git a/scripts/helm/helmcharts/versionUpdater.sh b/scripts/helmcharts/versionUpdater.sh similarity index 100% rename from scripts/helm/helmcharts/versionUpdater.sh rename to scripts/helmcharts/versionUpdater.sh From 7ce4365002715253644a0465bd367b824b431ef5 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 16 Jan 2022 21:46:58 +0100 Subject: [PATCH 115/206] Updated third-party --- third-party.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/third-party.md b/third-party.md index 03e3d8a90..fba7a49f5 100644 --- a/third-party.md +++ b/third-party.md @@ -28,8 +28,16 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan | pyjwt | MIT | Python | | jsbeautifier | MIT | Python | | psycopg2-binary | LGPL | Python | -| pytz | MIT | Python | +| fastapi | MIT | Python | +| uvicorn | BSD | Python | +| python-decouple | MIT | Python | +| pydantic | MIT | Python | +| apscheduler | MIT | Python | +| python-multipart | Apache | Python | +| elasticsearch-py | Apache2 | Python | +| jira | BSD2 | Python | | clickhouse-driver | MIT | Python | +| python3-saml | MIT | Python | | kubernetes | Apache2 | Python | | chalice | Apache2 | Python | | pandas | BSD3 | Python | @@ -80,11 +88,6 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan | source-map | BSD3 | JavaScript | | aws-sdk | Apache2 | JavaScript | | serverless | MIT | JavaScript | -| schedule | MIT | Python | -| croniter | MIT | Python | | lib/pq | MIT | Go | | peerjs | MIT | JavaScript | | antonmedv/finder | MIT | JavaScript | -| elasticsearch-py | Apache2 | Python | -| sentry-python | BSD2 | Python | -| jira | BSD2 | Python | From 9455c66620d046e64592bfd9639d963838bfd900 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 16 Jan 2022 21:47:16 +0100 Subject: [PATCH 116/206] Update third-party.md --- third-party.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party.md b/third-party.md index fba7a49f5..a26f646c3 100644 --- a/third-party.md +++ b/third-party.md @@ -1,4 +1,4 @@ -## Licenses (as of October 28, 2021) +## Licenses (as of January 16, 2022) Below is the list of dependencies used in OpenReplay software. Licenses may change between versions, so please keep this up to date with every new library you use. From bb497eed5d8df7a23fdff7dabfadfbaab0d51f9e Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 02:53:31 +0530 Subject: [PATCH 117/206] chore(install): remove the unnecessary validations Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/init.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/helmcharts/init.sh b/scripts/helmcharts/init.sh index c284fab92..2529254b0 100644 --- a/scripts/helmcharts/init.sh +++ b/scripts/helmcharts/init.sh @@ -70,16 +70,6 @@ randomPass() { ## Prepping the infra -## Don't override existing variables file. -[[ -f vars.yaml ]] && { - warn "Existing Variables file. Not downloading." -}|| { - -info " Downloading vars file" -curl -L -O vars.yaml https://raw.githubusercontent.com/rjshrjndrn/openreplay/${version="v1.4.0" - -} - [[ -z $DOMAIN_NAME ]] && { fatal 'DOMAIN_NAME variable is empty. Rerun the script `DOMAIN_NAME=openreplay.mycomp.org bash init.sh `' } From 51f49a7f02b8a37474a42e7285b56e2ad038cc6a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sun, 16 Jan 2022 22:44:01 +0100 Subject: [PATCH 118/206] feat(installation): alerts env vars --- .../charts/alerts/templates/deployment.yaml | 32 ++++++++++++++++--- .../openreplay/charts/alerts/values.yaml | 14 ++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index 3639ae413..a780e481f 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -38,12 +38,34 @@ spec: {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - - name: POSTGRES_STRING - value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:{{ .Values.global.postgresql.postgresqlPassword }}@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' - {{- range $key, $val := .Values.env }} - - name: {{ $key }} + - name: version_number + value: '{{ .Chart.AppVersion }}' + - name: pg_host + value: '{{ .Values.global.postgresql.postgresqlHost }}' + - name: pg_port + value: "5432" + - name: pg_dbname + value: "{{ .Values.global.postgresql.postgresqlDatabase }}" + - name: pg_user + value: '{{ .Values.global.postgresql.postgresqlUser }}' + - name: pg_password + value: '{{ .Values.global.postgresql.postgresqlPassword }}' + - name: S3_HOST + { { - if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" } } + value: 'https://{{ .Values.global.domainName }}' + { { - else } } + value: '{{ .Values.global.s3.endpoint }}' + { { - end } } + - name: S3_KEY + value: '{{ .Values.global.s3.accessKey }}' + - name: S3_SECRET + value: '{{ .Values.global.s3.secretKey }}' + - name: AWS_DEFAULT_REGION + value: '{{ .Values.global.s3.region }}' + { { - range $key, $val := .Values.env } } + - name: { { $key } } value: '{{ $val }}' - {{- end}} + { { - end } } ports: - name: http containerPort: {{ .Values.service.port }} diff --git a/scripts/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml index 9a353b057..36ae111e3 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/values.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/values.yaml @@ -76,8 +76,18 @@ autoscaling: # targetMemoryUtilizationPercentage: 80 env: - ALERT_NOTIFICATION_STRING: http://chalice-openreplay.app.svc.cluster.local:8000/alerts/notifications - CLICKHOUSE_STRING: 'tcp://clickhouse.db.svc.cluster.local:9000/default' + EMAIL_HOST: '' + EMAIL_PORT: '587' + EMAIL_USER: '' + EMAIL_PASSWORD: '' + EMAIL_USE_TLS: 'true' + EMAIL_USE_SSL: 'false' + EMAIL_SSL_KEY: '' + EMAIL_SSL_CERT: '' + EMAIL_FROM: OpenReplay + SITE_URL: '' + PYTHONUNBUFFERED: '0' + nodeSelector: {} From 2a36f8330884b7c064aa0c97bc7ef6ba838c4782 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 03:30:09 +0530 Subject: [PATCH 119/206] chore(helm): smtp variable injection Signed-off-by: Rajesh Rajendran --- .../charts/alerts/templates/deployment.yaml | 18 ++++++++++++++++++ .../openreplay/charts/alerts/values.yaml | 9 --------- .../charts/chalice/templates/deployment.yaml | 18 ++++++++++++++++++ scripts/helmcharts/vars.yaml | 19 ++++++++++--------- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index a780e481f..34ece7eeb 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -62,6 +62,24 @@ spec: value: '{{ .Values.global.s3.secretKey }}' - name: AWS_DEFAULT_REGION value: '{{ .Values.global.s3.region }}' + - name: EMAIL_HOST + value: '{{ .Values.global.email.emailHost }}' + - name: EMAIL_PORT + value: '{{ .Values.global.email.emailPort }}' + - name: EMAIL_USER + value: '{{ .Values.global.email.emailUser }}' + - name: EMAIL_PASSWORD + value: '{{ .Values.global.email.emailPassword }}' + - name: EMAIL_USE_TLS + value: '{{ .Values.global.email.emailUseTls }}' + - name: EMAIL_USE_SSL + value: '{{ .Values.global.email.emailUseSsl }}' + - name: EMAIL_SSL_KEY + value: '{{ .Values.global.email.emailSslKey }}' + - name: EMAIL_SSL_CERT + value: '{{ .Values.global.email.emailSslCert }}' + - name: EMAIL_FROM + value: '{{ .Values.global.email.emailFrom }}' { { - range $key, $val := .Values.env } } - name: { { $key } } value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml index 36ae111e3..590303a91 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/values.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/values.yaml @@ -76,15 +76,6 @@ autoscaling: # targetMemoryUtilizationPercentage: 80 env: - EMAIL_HOST: '' - EMAIL_PORT: '587' - EMAIL_USER: '' - EMAIL_PASSWORD: '' - EMAIL_USE_TLS: 'true' - EMAIL_USE_SSL: 'false' - EMAIL_SSL_KEY: '' - EMAIL_SSL_CERT: '' - EMAIL_FROM: OpenReplay SITE_URL: '' PYTHONUNBUFFERED: '0' diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 805873de7..dd67fa1c5 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -70,6 +70,24 @@ spec: value: '{{ .Values.global.s3.sourcemaps_bucket }}' - name: js_cache_bucket value: '{{ .Values.global.s3.assetsBucket }}' + - name: EMAIL_HOST + value: '{{ .Values.global.email.emailHost }}' + - name: EMAIL_PORT + value: '{{ .Values.global.email.emailPort }}' + - name: EMAIL_USER + value: '{{ .Values.global.email.emailUser }}' + - name: EMAIL_PASSWORD + value: '{{ .Values.global.email.emailPassword }}' + - name: EMAIL_USE_TLS + value: '{{ .Values.global.email.emailUseTls }}' + - name: EMAIL_USE_SSL + value: '{{ .Values.global.email.emailUseSsl }}' + - name: EMAIL_SSL_KEY + value: '{{ .Values.global.email.emailSslKey }}' + - name: EMAIL_SSL_CERT + value: '{{ .Values.global.email.emailSslCert }}' + - name: EMAIL_FROM + value: '{{ .Values.global.email.emailFrom }}' {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index ebf9888cb..869c252e6 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -48,6 +48,16 @@ global: # are same as minio.global.minio.accesskey and secretKey accessKey: "changeMeMinioAccessKey" secretKey: "changeMeMinioPassword" + email: + emailHost: '' + emailPort: '587' + emailUser: '' + emailPassword: '' + emailUseTls: 'true' + emailUseSsl: 'false' + emailSslKey: '' + emailSslCert: '' + emailFrom: OpenReplay enterpriseEditionLicense: "" domainName: "" @@ -55,15 +65,6 @@ global: chalice: env: jwt_secret: "SetARandomStringHere" - # EMAIL_HOST: '' - # EMAIL_PORT: '587' - # EMAIL_USER: '' - # EMAIL_PASSWORD: '' - # EMAIL_USE_TLS: 'true' - # EMAIL_USE_SSL: 'false' - # EMAIL_SSL_KEY: '' - # EMAIL_SSL_CERT: '' - # EMAIL_FROM: OpenReplay # SAML2_MD_URL: '' # idp_entityId: '' # idp_sso_url: '' From fb0b020c4aac5de8fac24e3060d7a0376cdac300 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 03:35:47 +0530 Subject: [PATCH 120/206] fix(helm): helm template error Signed-off-by: Rajesh Rajendran --- .../charts/alerts/templates/deployment.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index 34ece7eeb..21b7c9167 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -51,11 +51,11 @@ spec: - name: pg_password value: '{{ .Values.global.postgresql.postgresqlPassword }}' - name: S3_HOST - { { - if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" } } + {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} value: 'https://{{ .Values.global.domainName }}' - { { - else } } + {{- else }} value: '{{ .Values.global.s3.endpoint }}' - { { - end } } + {{- end }} - name: S3_KEY value: '{{ .Values.global.s3.accessKey }}' - name: S3_SECRET @@ -80,10 +80,10 @@ spec: value: '{{ .Values.global.email.emailSslCert }}' - name: EMAIL_FROM value: '{{ .Values.global.email.emailFrom }}' - { { - range $key, $val := .Values.env } } - - name: { { $key } } + {{- range $key, $val := .Values.env }} + - name: {{ $key }} value: '{{ $val }}' - { { - end } } + {{- end }} ports: - name: http containerPort: {{ .Values.service.port }} From 1d68dfe5e35bc681f3c1bfdc1b19400b392bc158 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 03:52:01 +0530 Subject: [PATCH 121/206] chore(helm): fix duplicate env keys in helm ref: https://stackoverflow.com/questions/60727150/helm-error-when-updating-upgrade-failed-the-order-in-patch-list Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/charts/chalice/values.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 509516136..4418bd704 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -81,15 +81,6 @@ env: captcha_server: '' captcha_key: '' async_Token: '' - EMAIL_HOST: '' - EMAIL_PORT: '587' - EMAIL_USER: '' - EMAIL_PASSWORD: '' - EMAIL_USE_TLS: 'true' - EMAIL_USE_SSL: 'false' - EMAIL_SSL_KEY: '' - EMAIL_SSL_CERT: '' - EMAIL_FROM: OpenReplay SITE_URL: '' announcement_url: '' jwt_secret: "SetARandomStringHere" From 4d456ce259f14d279b330f86079832c716128638 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:01:23 +0530 Subject: [PATCH 122/206] chore(helm): updated comment Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 869c252e6..ac093d6bf 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -1,7 +1,7 @@ fromVersion: "v1.4.0" # Databases specific variables postgresql: &postgres - # For genrating psswords + # For generating passwords # `openssl rand -hex 20` postgresqlPassword: "changeMePassword" postgresqlHost: "postgresql.db.svc.cluster.local" @@ -21,6 +21,8 @@ kafka: &kafka kafkaUseSsl: "false" redis: &redis + # For enterpriseEdition + # enabled: false redisHost: "redis-master.db.svc.cluster.local" redisPort: "6379" From 7a690329be49fed982723196f60a1574cd88d5e4 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:07:28 +0530 Subject: [PATCH 123/206] chore(helm): adding site_url variable Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/alerts/templates/deployment.yaml | 2 ++ scripts/helmcharts/openreplay/charts/alerts/values.yaml | 1 - .../openreplay/charts/chalice/templates/deployment.yaml | 2 ++ scripts/helmcharts/openreplay/charts/chalice/values.yaml | 1 - 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index 21b7c9167..6cdbe17b6 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -50,6 +50,8 @@ spec: value: '{{ .Values.global.postgresql.postgresqlUser }}' - name: pg_password value: '{{ .Values.global.postgresql.postgresqlPassword }}' + - name: SITE_URL + value: 'https://{{ .Values.global.domainName }}' - name: S3_HOST {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} value: 'https://{{ .Values.global.domainName }}' diff --git a/scripts/helmcharts/openreplay/charts/alerts/values.yaml b/scripts/helmcharts/openreplay/charts/alerts/values.yaml index 590303a91..294434fd8 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/values.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/values.yaml @@ -76,7 +76,6 @@ autoscaling: # targetMemoryUtilizationPercentage: 80 env: - SITE_URL: '' PYTHONUNBUFFERED: '0' diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index dd67fa1c5..ab68e07b7 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -50,6 +50,8 @@ spec: value: '{{ .Values.global.postgresql.postgresqlUser }}' - name: pg_password value: '{{ .Values.global.postgresql.postgresqlPassword }}' + - name: SITE_URL + value: 'https://{{ .Values.global.domainName }}' - name: S3_HOST {{- if eq .Values.global.s3.endpoint "http://minio.db.svc.cluster.local:9000" }} value: 'https://{{ .Values.global.domainName }}' diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 4418bd704..e5c579739 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -81,7 +81,6 @@ env: captcha_server: '' captcha_key: '' async_Token: '' - SITE_URL: '' announcement_url: '' jwt_secret: "SetARandomStringHere" jwt_algorithm: HS512 From c961ac97604f6ec1a495127b185906939e2e5c78 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:22:38 +0530 Subject: [PATCH 124/206] chore(helm): clickhouse env variable Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/charts/db/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/db/values.yaml b/scripts/helmcharts/openreplay/charts/db/values.yaml index 77acb5a80..70c181ce2 100644 --- a/scripts/helmcharts/openreplay/charts/db/values.yaml +++ b/scripts/helmcharts/openreplay/charts/db/values.yaml @@ -75,7 +75,8 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 -env: {} +env: + CLICKHOUSE_STRING: tcp://clickhouse.db.svc.cluster.local:9000/default nodeSelector: {} From 28d3d3b3fa70da1619cf93a849f1f5932fb13668 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:27:10 +0530 Subject: [PATCH 125/206] chore(helm): chalice EE key Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/chalice/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index ab68e07b7..41df5e2f5 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -38,6 +38,8 @@ spec: {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: + - name: LICENSE_KEY + value: '{{ .Values.global.enterpriseEditionLicense }}' - name: version_number value: '{{ .Chart.AppVersion }}' - name: pg_host From d9d338f23c0e6e910d27f97f9221dea8fda1727c Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:30:53 +0530 Subject: [PATCH 126/206] chore(cli): making variable optional Signed-off-by: Rajesh Rajendran --- scripts/helm/openreplay-cli | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/helm/openreplay-cli b/scripts/helm/openreplay-cli index 6fa9472f6..b9db53b6c 100755 --- a/scripts/helm/openreplay-cli +++ b/scripts/helm/openreplay-cli @@ -109,7 +109,7 @@ legacyInstall() { } helmInstall() { - helm upgrade --install openreplay openreplay -f vars.yaml + helm upgrade --install openreplay -n app openreplay -f vars.yaml } upgrade() { @@ -127,7 +127,7 @@ status() { [[ $# -eq 0 ]] && usage && exit 1 -PARSED_ARGUMENTS=$(color getopt -a -n openreplay-cli -o vhds:S:l:r:i:I: --long verbose,help,status,start:,stop:,logs:,restart:,legacy-install:,helm-install: -- "$@") +PARSED_ARGUMENTS=$(color getopt -a -n openreplay-cli -o vhds:S:l:r:i:I --long verbose,help,status,start:,stop:,logs:,restart:,legacy-install:,helm-install -- "$@") VALID_ARGUMENTS=$? if [[ "$VALID_ARGUMENTS" != "0" ]]; then usage @@ -140,12 +140,12 @@ do -v | --verbose) VERBOSE=1 ; shift ;; -h | --help) usage ; shift ;; -d | --status) status ; shift ;; + -I | --helm-install) helmInstall; shift ;; -s | --stop) stop $2 ; shift 2 ;; -S | --start) start $2 ; shift 2 ;; -l | --logs) logs "$2" ; shift 2 ;; -r | --restart) restart "$2" ; shift 2 ;; -i | --legacy-install) legacyInstall "$2" ; shift 2 ;; - -I | --helm-install) helmInstall "$2" ; shift 2 ;; # -- means the end of the arguments; drop this, and break out of the while loop --) shift; break ;; # If invalid options were passed, then getopt should have reported an error, From b2bedd39237296d68140df9abb02651176712612 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:33:47 +0530 Subject: [PATCH 127/206] chore(helm): fix variable precidence Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/templates/job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index aa1b4467d..88962a624 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -123,7 +123,7 @@ spec: - name: KAFKA_SSL value: "{{ .Values.global.kafka.kafkaUseSsl }}" - name: PREVIOUS_APP_VERSION - value: "{{ .Values.global.fromVersion }}" + value: "{{ .Values.fromVersion }}" - name: CHART_APP_VERSION value: "{{ .Chart.AppVersion }}" command: From a74f6bdfbe797b28d2cf739b5cbe6517bc86763b Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 04:39:17 +0530 Subject: [PATCH 128/206] chore(helm): removing depricated docs --- scripts/README.md | 33 ----------------------------- scripts/helm/README.md | 47 ------------------------------------------ 2 files changed, 80 deletions(-) delete mode 100644 scripts/README.md delete mode 100644 scripts/helm/README.md diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index be8b97b11..000000000 --- a/scripts/README.md +++ /dev/null @@ -1,33 +0,0 @@ -### Installing OpenReplay on any VM (Debian based, preferably Ubuntu 20.04) - -You can start testing OpenReplay by installing it on any VM (at least `2 vCPUs, 8 GB of RAM and 50 GB of storage`). We'll initialize a single node kubernetes cluster with [k3s](https://k3s.io) and install OpenReplay on the cluster. - -```bash -cd helm && bash install.sh -``` - -### Installing OpenReplay on Kubernetes - -OpenReplay runs 100% on kubernetes. So if you've got a kubernetes cluster, preferably, a cluster dedicated to OpenReplay (on a single node of `4 vCPUs, 8 GB of RAM and 50 GB of storage`). You can run the script, which internally uses helm to install OpenReplay. - -We hope your cluster has provision to create a [service type](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) `LoadBalancer` for exposing OpenReplay on the internet. - -```bash -cd helm && bash kube-install.sh -``` - -### OpenReplay CLI - -The CLI is helpful for managing basic aspects of your OpenReplay instance, things such as restarting or reinstalling a service, accessing a component's logs or simply checking the status of your backend services. Below the list of covered operations: - - - status: status of the running services - - logs: logs of a specific service - - stop: stop one or all services - - start: start one or all services - - restart: restart one or all services - - For more information: - - ```bash - cd helm && openreplay-cli -h - ``` diff --git a/scripts/helm/README.md b/scripts/helm/README.md deleted file mode 100644 index 2ac1a3556..000000000 --- a/scripts/helm/README.md +++ /dev/null @@ -1,47 +0,0 @@ -## Helm charts for installing OpenReplay components - -Installation components are separated by namespaces. - -**Namespace:** - -- **app:** Core OpenReplay application related components. - - alerts - - utilities - - assets - - chalice - - ender - - sink - - storage - - http - - integrations - - db - -- **db:** Contains following databases and backend components. - - kafka (ee) - - redis - - postgresql - - clickhouse (ee) - - minio - - nfs-server - -- **longhorn:** Storage solution for kubernetes PVs. - -- **nginx-ingress:** Nginx ingress for internet traffic to enter the kubernetes cluster. - -**Scripts:** -- **install.sh** - - Installs OpenReplay in a single node machine. - - This script is a wrapper around the `install.sh` with [k3s](https://k3s.io/) as kubernetes distro. - - Note: As of now this script support only Ubuntu, as we've to install some packages to enable `NFS`. - -- **kube-install.sh:** - - Installs OpenReplay on any given kubernetes cluster. Has 3 configuration types: - - small (2cores 8G RAM) - - medium (4cores 16G RAM) - - recommended (8cores 32G RAM) - - For all options, `bash kube-install.sh -h` From cae6eaea7e51d298c1117529e482f172bb33fa62 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 06:13:58 +0000 Subject: [PATCH 129/206] Update README.md --- scripts/helmcharts/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/helmcharts/README.md b/scripts/helmcharts/README.md index ee63b1567..485340458 100644 --- a/scripts/helmcharts/README.md +++ b/scripts/helmcharts/README.md @@ -11,4 +11,5 @@ ## Installation helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic + helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./values.yaml --atomic From 57af0c5278d16d4723f4e213b43de7d8c824fbde Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 06:14:36 +0000 Subject: [PATCH 130/206] Update README.md --- scripts/helmcharts/README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scripts/helmcharts/README.md b/scripts/helmcharts/README.md index 485340458..b54ca9652 100644 --- a/scripts/helmcharts/README.md +++ b/scripts/helmcharts/README.md @@ -1,13 +1,5 @@ - Initialize databases - we've to pass the --wait flag, else the db installation won't be complete. and it'll break the db init. - - collate all dbs required - - How to distinguish b/w enterprise and community - - Or fist only community then enterprise -- install db migration - - have to have another helm chart with low hook value for higher prioriry -- install app - - customize values.yaml file - ## Installation helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic From fa8109bb37c365b9afcdfa9c152ccfc9294d8ef1 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Mon, 17 Jan 2022 11:37:19 +0100 Subject: [PATCH 131/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index ac093d6bf..50bf9edd2 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -59,7 +59,7 @@ global: emailUseSsl: 'false' emailSslKey: '' emailSslCert: '' - emailFrom: OpenReplay + emailFrom: 'OpenReplay' enterpriseEditionLicense: "" domainName: "" From a57bae50dc9c3b5134b1da47d729983b162febae Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 11:46:45 +0100 Subject: [PATCH 132/206] feat(installation): create missing mobile bucket --- scripts/helmcharts/openreplay/files/minio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/files/minio.sh b/scripts/helmcharts/openreplay/files/minio.sh index 2bb7682f3..dc8fbdbb0 100644 --- a/scripts/helmcharts/openreplay/files/minio.sh +++ b/scripts/helmcharts/openreplay/files/minio.sh @@ -5,7 +5,7 @@ set -e cd /tmp -buckets=("mobs" "sessions-assets" "static" "sourcemaps") +buckets=("mobs" "sessions-assets" "static" "sourcemaps" "sessions-mobile-assets") mc alias set minio http://minio.db.svc.cluster.local:9000 $MINIO_ACCESS_KEY $MINIO_SECRET_KEY From 948e6fdfc9c19248970465e3bf78974ff5d8c1d7 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 11:50:15 +0100 Subject: [PATCH 133/206] feat(installation): fixed vars.yaml emailFrom feat(installation): removed unused certificates --- .../helm/roles/openreplay/asayer.local.crt | 30 ----------- .../helm/roles/openreplay/asayer.local.pem | 52 ------------------- .../roles/openreplay/openreplay.local.crt | 30 ----------- .../roles/openreplay/openreplay.local.pem | 52 ------------------- scripts/helmcharts/vars.yaml | 2 +- 5 files changed, 1 insertion(+), 165 deletions(-) delete mode 100644 scripts/helm/roles/openreplay/asayer.local.crt delete mode 100644 scripts/helm/roles/openreplay/asayer.local.pem delete mode 100644 scripts/helm/roles/openreplay/openreplay.local.crt delete mode 100644 scripts/helm/roles/openreplay/openreplay.local.pem diff --git a/scripts/helm/roles/openreplay/asayer.local.crt b/scripts/helm/roles/openreplay/asayer.local.crt deleted file mode 100644 index 49462c11e..000000000 --- a/scripts/helm/roles/openreplay/asayer.local.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFGTCCAwGgAwIBAgIUH3ILXUbpzJ8NxM1al5AIq1lcR10wDQYJKoZIhvcNAQEL -BQAwHDEaMBgGA1UEAwwRYXNheWVyLmxvY2FsLmhvc3QwHhcNMjEwNDA5MTAwNjUx -WhcNMjIwNDA5MTAwNjUxWjAcMRowGAYDVQQDDBFhc2F5ZXIubG9jYWwuaG9zdDCC -AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALH673udMEJHrhTm5LdE1/Wy -Q8cD/RTxRwGei3rP4AmWEZrbD7DpgT+DDM5U3JgH7MN7PwDN3taa4cOErJNJzn6l -YQyips/xVMokVi9pcvXdLRElc8xLbF2u/O1JhefsLUFyYO83U7vyFmNy1ARN+SGU -SMMPa0BC9H1g2KsOKS7R5JAJtiNc5Z8bA7NdvZKYeG5AC+9q5mvAQ21CCm0lK2KA -P/txMVd4+Pu6uywqKxcmzLFtHM9AVr2IkobopxzocPdJh1ypqaTIWWvYqo4Iylv9 -DqEwlceRZUd5rOGcmeekptNAqy57U47ohnD8Lf7ZvkMNn3aBUxhI6nl6ZFcZAi23 -BIkYnhT4cQPloih2Z9bnr4HHdHofGAGrBbv0Oqgbxfu5GgCKM11LM10YKgSOEw8I -TG+1E08MVDt4Gxe4NfDDJC9UpkXAb+6b7vuZm4dNbDPRRz6JtUh+n0H+dmBNA+1x -P3FeXO206VuTWlj5jcndKKY0lvcb10i5StYWA1AVxR7TSR6K6lVFkMPWMCY5bNrY -BEzA8A3rCVe5UVdQvcBrovftafD67cSxC5Gz04Xc7H+M2hUBpk/AooTOxunHDcFz -yuvbVQW6L6aM1YOytup2tTOILYcKaPTiUf4J70ypUhlPaJ1qUxK1hDV+gHbSEUlx -KGoJAlg+KGXTt1vzPUphAgMBAAGjUzBRMB0GA1UdDgQWBBRihXjPkaSNtnQW8o1j -bXztF0iK3jAfBgNVHSMEGDAWgBRihXjPkaSNtnQW8o1jbXztF0iK3jAPBgNVHRMB -Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAS1JO5tx7AKY2wh/gP6B160ETP -m3P/QWPWkLpAOIPZnk2aFC3AJvNnqfzeq4LHL19xbJ19SZJpuqi/E2DqPCOLmFwT -mTbfjZEmmYkmYqggd20v8TsOgxwF8Uz5oIJwSxPSJnFPFDVsbZrHR9OwPdVgjzr+ -helqNuw9gqyKxq+2sXKxGdMnoNlSfCJXse+3cgIsh6AXmG/XJQ4AldLSQkZvyddX -MVAImYv7oCynVJ4tsLseqmi4fzOj2Gkk8n5JFkvtmXi6e1F6Kgj+7CzTTB+tQiNu -gWDyySK4P9H7ASRcVAoCdwkFjiT9Yd3D8AcykeuP2TW+22PedL2eqEnhFrYxSe7x -jeZzPWm1Rkn8hhzXjm8ks2Mc9m4S+ncCnKKIVRHqmDmhyXZG/6gbwS/JCO9u2qiJ -9xzgNsiRS5VEgQfZy7SIaY8armCKNsUDp8ZxvWpextbBJzEVebLCF/MeMRTMTOMg -P02CHraJDXA64KBrmq1Y394itMxhqpZySrW0Fa7sLXkiLyQY8T0xX08JlUmerRls -msGM58FwZJ8kJ8HXxMs//Gt8S3OtiGjO7Ro/yduTKKjobBTDGWbjVKMzuz6Ly6yD -7Z7cHebqJvQODyFYsswVNV7pVfoj4HdhEybKUlJzGjDkk99wktBExsTD1TAkTP79 -olSG5Nko3vBfgRmuNQ== ------END CERTIFICATE----- diff --git a/scripts/helm/roles/openreplay/asayer.local.pem b/scripts/helm/roles/openreplay/asayer.local.pem deleted file mode 100644 index 805a3a93b..000000000 --- a/scripts/helm/roles/openreplay/asayer.local.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCx+u97nTBCR64U -5uS3RNf1skPHA/0U8UcBnot6z+AJlhGa2w+w6YE/gwzOVNyYB+zDez8Azd7WmuHD -hKyTSc5+pWEMoqbP8VTKJFYvaXL13S0RJXPMS2xdrvztSYXn7C1BcmDvN1O78hZj -ctQETfkhlEjDD2tAQvR9YNirDiku0eSQCbYjXOWfGwOzXb2SmHhuQAvvauZrwENt -QgptJStigD/7cTFXePj7urssKisXJsyxbRzPQFa9iJKG6Kcc6HD3SYdcqamkyFlr -2KqOCMpb/Q6hMJXHkWVHeazhnJnnpKbTQKsue1OO6IZw/C3+2b5DDZ92gVMYSOp5 -emRXGQIttwSJGJ4U+HED5aIodmfW56+Bx3R6HxgBqwW79DqoG8X7uRoAijNdSzNd -GCoEjhMPCExvtRNPDFQ7eBsXuDXwwyQvVKZFwG/um+77mZuHTWwz0Uc+ibVIfp9B -/nZgTQPtcT9xXlzttOlbk1pY+Y3J3SimNJb3G9dIuUrWFgNQFcUe00keiupVRZDD -1jAmOWza2ARMwPAN6wlXuVFXUL3Aa6L37Wnw+u3EsQuRs9OF3Ox/jNoVAaZPwKKE -zsbpxw3Bc8rr21UFui+mjNWDsrbqdrUziC2HCmj04lH+Ce9MqVIZT2idalMStYQ1 -foB20hFJcShqCQJYPihl07db8z1KYQIDAQABAoICADXI7GhBx1ywRrZAVGvh3RV9 -yqew1+FlTJz7qZDykvSyqh1PB/3hgtKLquoLkSJMestUSmTW0fmukn1AcdqM2l9B -YQOtf5Wli4cO1hC74CbWsRDWbZkWTeg9wmd+6X19zOtTQmHoKWDJtuRUxieK3Le+ -FQMFfNbNpQ7inneK7v0cJuM0XBtxrl6XjlW9BVXwx6whpLm0Z2+2iWnSJE55hhPd -R/iXa17QywVfl8Kv95emfQM9QxCKk+vxAr7MPfwpbkH8sXBaQU7v0L4bmucTVvvX -jH0/rnPBx4aDCbCgX442HsgGafTBBi5QrJgmTUwPx0SCMGdii5byKhGN4BbGEvb2 -I5NaFXgGHxm4ObMUeVgRnr5ym5e8J8wykDzM/VJRqxdav3kRtUcLMWzhoK57kf0C -BkgQBp0DX6kWWEWrLaioRfQbyOGYo7ko547sHAYK54C0EUjqSdCMh4Sl1RR7B+yw -uAoi5qqDeh9D0k5LwRgI7+jS5t7bLgxlLQeXYgtj9g27nz8Q/w9P2/pSf6xVOC9T -oin48ynM8BsLoQGp/+pg3E091gSGHFjrsLTxCvMN/au3k3onI728Fm1dyjNQF54g -486W7za8hbf3SZhKmRFCDM+ePXqQjEWZqH65YRB+9BsP3gmCWoPsoZAgZ3ZC3GNJ -C1FUQkD3kqhcw7C/SthxAoIBAQDp5Rw3uN4kps4sKSCTQqKgHxFj2mR+N2+xgDib -LchvYqh2OLUo/2u0p4+X9/PYfIvhh+gPca7yTp5m3JUxTuFHgjScl/cDqrWkY7mn -7kLnU5qy67V9ouknY+64MUa3w08v8oIz+l0Kv/YWIztM66yY3mMcBaYix18njqM8 -wuXXjXfHs5WRyozNw2L6/ZOf0foNPKuKssPBGFD3rk6R812EbLfOHn8dLjoePolv -KAWCpK1fZcOYHfUCldPxM+vVtQ4/mMauAxYc6vCKC+MCWWKEU0vhuswITAg+r/Cb -gbjlFUlTO9LZ99a9WIW072GyZG63nPo66Fbz5yoBPpbSsdP9AoIBAQDCzQP5qfkI -g3e/FxYainJgbGMv4mNlCSdEAjStVt8KRwEHJq8Je34TMtuW5joRweJ0cwsF7zhH -qv8B/+9OfTlqdYWWMJhsBHJCi2BMdP+K9D4FgD7YJsJFqhYNYtAVjQpcLV7pbyzw -Q6cM+KFGAiudP2uVLbLSgUrFJj8su40hXFp9Ey4ZmwqtF7APj5p8fSdyuKqe3GZA -M4tAaiflnOUO6XFUcXShdn7z1Bg8MR3RfOl3kdg1tquIQyRygCT9DvD1OJofRaJ5 -prxU5kBZRy8NkKMDJwMpN2HNNv6A4E9tKPiFfKS2IDx8DHKVQqokzGeqJFjKD6X7 -mtMnlscURTM1AoIBAEN3auRH1wwLcX0bf+H6Xp6BV6rN3B7sLrOsoKdiVWaLtrwj -L15cDt1VV/4l75iKlG7BnRaXXDdYpihViFNoxWKzFCn3S7ErFKoOegcOmw3O99iU -VQ2gaFCMateGnzdyhuz1rsj/dscbCAjqsLhHYsnVnBEBMCLoD3KBmmR0c4fxrpNU -Idij7GmN6ocrjeGT1+TpkzG2zvBr6y8GgHNpF5iTI5G5CkGIjAsdTtdZW/2LMtdJ -q1m9imbsgO4aY/SI/Usq7mRTT5WXzzqCo0CSN/CjdkfucdwS0m4Cogy5JnQt57Cy -mOd2reKAHlFKm4uxnJ4qEaPcUPiZvWwOecLcuKECggEBAJQGRecJifs2K2BBosI4 -QgGTWyuf0/eKpRPAuVaelI2yts1AT/HcrUjZdJOODsI/mlzyroWr6hBLl8tlN+tk -rKSA+SnPc4NhPZHjZyGyJhzoy9hSQ27sxOM7A9bYJ1a4/g5dS6oGmRdZ0S7CH4qb -S5kKAC4zs5bSOIbnMUpEmlGnY/t2/6jlb49oWJj6fmHqmHdMbBqbT/epMz+XawnH -1yclrMCGFiI7cOzHVkMm74moaaA6axkl3eKFYsS/K1xAZklw4Wyx9WyLjsDU1kEB -HngoWDalBnnjY6YVC//Sc4h3gaGtvVvH0y5QJyHUroWX7Y4UYIPqsC4o0BtdL5nv -m6ECggEBAK15GYZSh9DE1YBt+507tmNbMFPxYnbx6ued/pNunA2uP6r/B+Ky5voG -kwFbBM73xTIIeweFG8WrYK5p/9ESfj4csVvFnwk4NDWwDquKKCeO1POMVYFcu+hl -si/mGdsIaA83LuWI4uGCVNmknaHNQ9Qr5QixMO7RHPYD4Ktej+HEGdiOt5g9L9Px -9rczzA8xiqalOXgLYzksiwiiPSpsMCKDcgLIwqNeaHTz9N2WFmG9MALjjJkyGAdD -OJ93XooOqww1OIPZQTK1bgEBZjevuzlMoI5aD68W94Vbkpy8GebcFSnrQkd/8Qn9 -FqR3uiHJh3vCo3I9vsS1ewNQgUgmGOk= ------END PRIVATE KEY----- diff --git a/scripts/helm/roles/openreplay/openreplay.local.crt b/scripts/helm/roles/openreplay/openreplay.local.crt deleted file mode 100644 index d4f9e5ce3..000000000 --- a/scripts/helm/roles/openreplay/openreplay.local.crt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFITCCAwmgAwIBAgIUbXel1z1h6hUtPLFlRfmYDjiLoXkwDQYJKoZIhvcNAQEL -BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMDUwOTEz -MjkzOVoXDTIyMDUwOTEzMjkzOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh -bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAsH4BQXN8bDIR -D9RO59KIpKK4Qv1oppZ2k/ZbOXBzlYqLmWfdbarAMJDoQZvrwbPUlDDvudHwVeGt -HjW6jB9H0JFBZ20fhH0Uxf+4OKQ+r59IJMo8iPSQ03ZOA74KuHga1MpSZa1VB1hZ -dfZWL21vyVcllP4jMLNEfB0L63S3k0pSCHfatDKeQmMIkUO0zFZcsjcxHmZEGfnt -rWTvE7sQoJdwzTq59Bpxpylvga48Mex1pU9c4utdMMLazMBlPCP26P8OKYu5LXN9 -0nNbEGFoolMbL8lUa63vNB8u1i+26+ESNcq/6T4R2VGcvg7qb5C90Spo6pOJVbmh -mX3I6ZQ8lP4Svv3VGb4nYObFf+Obu5X8c6w69f+AExwasYlrVnXDFCXVK8QGbvFE -WCBEVOYMJRA/zXbOi6VVO4PkFuAV8zIqWuGr++tb1sjdHg/xy6c/pahbqjG+mZ2m -oOpD5hmiYrQ8PPdDpSnufY/B86ldB6it5/cJgnC9VrZzRXUEOvCurx0a977SG5wh -Qg6Pnc309NCJMW/5vsLDK9/OPsOIsCm0JF9hNgqyzox2kyBacML0pws9Pn9ldeqF -dmyaQ98oIWSpTp/eMx2dJwPloEaom/e4tMxiLrt7zeHwWvv3UZNvDorijnshY66H -fq1gbhb4OCVOIQHSLqGMqQMBObBPIpUCAwEAAaNTMFEwHQYDVR0OBBYEFCjsH7qJ -jYUa5zKxggCdFW5Kqo1kMB8GA1UdIwQYMBaAFCjsH7qJjYUa5zKxggCdFW5Kqo1k -MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAIVPXyyScg1vILqZ -gJcJKskRqPAmKnwP1nBFw9uvCm5Js75nU/DhzCk6QGdevuSwDIXqquIlmvMzkubF -XZgZXEW2oq801h74jVe2ZFghvUOPchcS4ROXwHJRwORrYAidg0WFzG+HDD9qxuuK -T3J1t02O/O8EZ55BOgJXkHcSTfrVoRzkvOGgry243tnXAAeJFAeuQMEhpEianzFF -KDWykEQz7EnFRMqoaD5dvZ4IiHzPrsZ43yoewBOSwDvp/uXXhFgS4vJ0vEOZW2/L -JZd8/kK5JvxaLan3n7ALkLMH4ojQ9OIxWIY3aUoj/Oy/QnCK2vpzWIwwjpjLOxVF -UNhXZGwW4FbCtUPp27ctlEokjFmRvH6fQOGUhF5KCOXWC3zzwq3PnLCdBQjlG3by -NlcI5aIWUmKrx2xDcbtkJclqXxFsWx9kHUMurJ5Vf1ZVOh1+mAOy0F6+72AQN7AO -8YtzVjpdj29Sl3WbJx6DHHUD2T/EgPtkXiAu107Fd+tQbxO+Fp0X6oiAfRn/7nq0 -fQGlQbtleIPSjepHBEYwjCpRxtEPdvRj93w/M8Q9RW8w9shhGO6QElCuL3okOqNB -YMP3NSOjAInDwqn1aSj/R+G4S2HvaWP2n3wJONFxhxgFISoaTOBXuH/dQySDICq0 -vy50cJ+11ukSuYOwIKJGaKzlsUvg ------END CERTIFICATE----- diff --git a/scripts/helm/roles/openreplay/openreplay.local.pem b/scripts/helm/roles/openreplay/openreplay.local.pem deleted file mode 100644 index aeaafcc13..000000000 --- a/scripts/helm/roles/openreplay/openreplay.local.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCwfgFBc3xsMhEP -1E7n0oikorhC/WimlnaT9ls5cHOViouZZ91tqsAwkOhBm+vBs9SUMO+50fBV4a0e -NbqMH0fQkUFnbR+EfRTF/7g4pD6vn0gkyjyI9JDTdk4Dvgq4eBrUylJlrVUHWFl1 -9lYvbW/JVyWU/iMws0R8HQvrdLeTSlIId9q0Mp5CYwiRQ7TMVlyyNzEeZkQZ+e2t -ZO8TuxCgl3DNOrn0GnGnKW+Brjwx7HWlT1zi610wwtrMwGU8I/bo/w4pi7ktc33S -c1sQYWiiUxsvyVRrre80Hy7WL7br4RI1yr/pPhHZUZy+DupvkL3RKmjqk4lVuaGZ -fcjplDyU/hK+/dUZvidg5sV/45u7lfxzrDr1/4ATHBqxiWtWdcMUJdUrxAZu8URY -IERU5gwlED/Nds6LpVU7g+QW4BXzMipa4av761vWyN0eD/HLpz+lqFuqMb6Znaag -6kPmGaJitDw890OlKe59j8HzqV0HqK3n9wmCcL1WtnNFdQQ68K6vHRr3vtIbnCFC -Do+dzfT00Ikxb/m+wsMr384+w4iwKbQkX2E2CrLOjHaTIFpwwvSnCz0+f2V16oV2 -bJpD3yghZKlOn94zHZ0nA+WgRqib97i0zGIuu3vN4fBa+/dRk28OiuKOeyFjrod+ -rWBuFvg4JU4hAdIuoYypAwE5sE8ilQIDAQABAoICAAYuM28OU0IasrxCLA00MzLI -u/kklKCYzkg6LVGinXmUI3VYzMlFXHuW53s8sZHT8VLLL+nq7yaiw4Q2T3UOt14X -aBnuGVdzZeHVJadeIQ/XXRhkuJ/3KIcvMmYxJoD1O8NN3nrPChY6ws89tCDvDSYY -1ikUiTjwXiqWfc9eA/u1c4/2WoNK7n3Wp/smStOJBq/6/M5auHHMbcVProubVma5 -Rur47oAc2PQIoG4DIvqAlFIYqGEN6vKFMFJRPnpPrBrWq7FKwk5UKAjRn1PAhYCB -tocC3uDyc1j46pP3kclK1W9qzTOmx8s7b2i8bvs/L1Z+zLPCPOHyoTUV0Y5UcZab -LzjX7DhPtX+wDX6TMnYe1dDcSa0Dt1bf98qSMJ3VCgim2VNqOHiv1BINJLbP+OBx -ezqc+MCDX/PgyCe0m20pKKVEIGh3U9nnwMOKTJkeZDj2gO4O0aUT8TQw61INHSSQ -2MGFTOnIk2WOu/N1+gYTWsU49GoNyXoNXRIkRIxHGvfCuIjDlIrPIp9id+pBJbze -ajRYyJYa2M1iHXH8HTgfSVrhB+H3MYooviUyRK/YSBDxeU9RKnNDDMRjYhIm74Fy -gDH5sTuD/JC/mPVP8xvu9+FY5Pp77rTZ5oJx2uKeYSqBK4jYyzgeZI6NNJvVMRN3 -tdIufHm9B5/nwWULRnDVAoIBAQDjdrYYLZLtvP7SLPpa23eAy0HOfSQeoCBZQVfd -LTwkspRegd9IOBGao5URsRBwtVqw78/+2O1gG8k1lgDBOEvVyB5/T2WcSesd6v5M -VswtmWA9KOVKDGpW/WWjOlOuvJKCfd1zyAHQQ0LVWltr2hIrxfooAjwj2vR/8C0t -jtDMsOGi1F1y4ji+X73PehKyR0kCOoP/0XwTFtgjzzRpogy3KNRZhwpGh1anuWHV -M4eFnCME4BLk6Q+2MmmuTAOyqs737mtyqMegoYl/8NauXsZo6J0PSugbjJDYGyvP -m/bzSYk2IDUx2iaCApbE6xsGegNmX9akuXDBxBAXYwAwYGuTAoIBAQDGokc8UbiS -aqxrsYy1JILZvquNXjDC1EgPaEsP8c1jQvPpdhJrFjYKW2SqGQRamjriFPimUnuQ -sIziNmJVZcRMOyWncgywhsuyre+uLEv/Wykdy/jNZ3W1SLqJ6I6wJYrQNz+3albG -S55vuR0ZwzQ/IE6pekdII+EryjSTAq8A8DV5WbwVIiZ/5ZmKrfKBTGaEraFkc8pH -YyBumhRT91gYzqsdFJ81ce01k5elvbW1YU2HnlZF2QwEzV4n46nrYmV/nw8jex0L -3Yb2/mP8aRSSvN8ZFKsG3voK6HGAwtEPDFL8aOLl/oUqNn6iY5WRkBduztipxH+v -qpk2brpfh6I3AoIBABqMQ2qE/y4SLeeX6kxqawM0NeUExA9y/vOL5dZhrjBWS2zQ -WaATMzLKPt7GvLV65S/bfXowFVxvQceoT0Q97tPvvGd9e82G7xMZ1z5PrxrvWMUq -ncBLvKmUf6VSbGvrhGoIiK03vcZx4z4FgsAOvqovPCgpQp2os3qvLIaxsXsCdgNM -J+/9qUPG5t9tTvtRyJqi+78C0yljNCzRxTtC95r0//vHABMFPr7qqxl4Hywf8hJi -mLHw5a2NKrp2EsRqnSdgw1epjUsZL3QoRtYpJ50c8R/UzUxR1qHyYeWkPWldvs77 -aVn9LRtlc/Vdv1S9LuJm8yVco2VqHbqtClubFVMCggEBAIxcA4ZmIMylwGzhK5Kc -2DHRgBwHIOGra7gndFMyBHZNy1l/a4hsS4eKCVoBD4iOT2mOdB3jTxlxCDyZPWcm -4E0VxofKGZlrxujd6+3hs1ogKq+5gHh00UJmwBnPUXS7MpnrUITlCahZqv07i4q2 -22Z4B93OYCxj3is5WWlAjjVViBPWCwfL9/SMY7ERNbmUKSU8JD19bMSBEVvWWOL0 -dzYgY93Kwq3bcUlYn+IoagxfU4+vnTsCInIKRKxfPaTtBNfnp6TkzCOkSEikfjbQ -cg82k1d/uVLDoWCv2/SmtV+yz7k4zIzFC4hGtbk+MWSIs2ZDl+puxeN144oJOH1k -tIUCggEAEKTCMtYFjNT+AdLnBUqS7r1UQ/z34Jmx8HF9jGSeE8WblNPxt7HPsDpw -d9ahwf5oLMyRNZe9TCpZH+BDGMTutQbPqmhqHPwPUfgUpqY3FKD6v46DIfoJ4k9w -vrqoS+FVJmFqwvvN5Z1Z61f2b6mEVXb5mBHogsk0rhCHP44aIPNB8h3Al3xX2UNE -GheUuGxdcEAhWzaGqAw5Y/TtgA8FFa8M7lL1xZcDXMW778oE1/Cu1y//6fJW5ssz -YLClXiE7AgTPOW/1kpI+YBauxXdcCeqDeHuBPzZ/FtzlzL3g24E84yEZvb9dzZW8 -h0BrE8NFZlEyg2ntATIT1MWhpIlsdA== ------END PRIVATE KEY----- diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index ac093d6bf..50bf9edd2 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -59,7 +59,7 @@ global: emailUseSsl: 'false' emailSslKey: '' emailSslCert: '' - emailFrom: OpenReplay + emailFrom: 'OpenReplay' enterpriseEditionLicense: "" domainName: "" From 4b927b859e268534a92fabd5c2263eb8ede58778 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 11:56:22 +0100 Subject: [PATCH 134/206] chore(helmchart): fixed typo in job.yaml --- scripts/helmcharts/openreplay/templates/job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index 88962a624..74e2cadcc 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -36,7 +36,7 @@ spec: args: - | git clone https://github.com/openreplay/openreplay /opt/openreplay/openreplay -b {{ .Chart.AppVersion }} - # Checking variable is empty. shell indipendant method + # Checking variable is empty. Shell independant method. if [ "x$ENTERPRISE_EDITION_LICENSE" != "x" ]; then echo "Copying enterprise code" cd /opt/openreplay/openreplay From b6dd8622fd18bed5c552452a9fd083dfcc01602e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Mon, 17 Jan 2022 12:09:33 +0100 Subject: [PATCH 135/206] Update vars.yaml --- scripts/helm/vars.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helm/vars.yaml b/scripts/helm/vars.yaml index f106f03af..f17ec904c 100644 --- a/scripts/helm/vars.yaml +++ b/scripts/helm/vars.yaml @@ -24,8 +24,8 @@ domain_name: "" docker_registry_username: "" docker_registry_password: "" docker_registry_url: "rg.fr-par.scw.cloud/foss" -image_tag: "v1.3.6" -openreplay_version: "v1.3.6" +image_tag: "v1.4.0" +openreplay_version: "v1.4.0" # Nginx ssl certificates. # in cert format From 234e7092474a558c9fdc531381f48dae54982622 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 17 Jan 2022 17:10:28 +0530 Subject: [PATCH 136/206] fix(install): ignore clickhouse db exist error Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/files/clickhouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/files/clickhouse.sh b/scripts/helmcharts/openreplay/files/clickhouse.sh index 80cb98fa3..503fd97bf 100644 --- a/scripts/helmcharts/openreplay/files/clickhouse.sh +++ b/scripts/helmcharts/openreplay/files/clickhouse.sh @@ -19,7 +19,7 @@ function init() { echo "Initializing clickhouse" for file in `ls ${clickhousedir}/create/*.sql`; do echo "Injecting $file" - clickhouse-client -h clickhouse.db.svc.cluster.local --port 9000 < $file + clickhouse-client -h clickhouse.db.svc.cluster.local --port 9000 < $file || true done } From 0f4ceb1ca72448265e933a9680c837ffd48040f7 Mon Sep 17 00:00:00 2001 From: Shekar Siri Date: Mon, 17 Jan 2022 18:09:30 +0530 Subject: [PATCH 137/206] fix(ui) - login redirect --- frontend/app/Router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/app/Router.js b/frontend/app/Router.js index 7e5511231..edacf56c5 100644 --- a/frontend/app/Router.js +++ b/frontend/app/Router.js @@ -102,7 +102,7 @@ class Router extends React.Component { this.props.fetchTenants(); } - if (!prevProps.isLoggedIn && this.props.isLoggedIn && this.state.destinationPath !== routes.login()) { + if (!prevProps.isLoggedIn && this.props.isLoggedIn && this.state.destinationPath !== routes.login() && this.state.destinationPath !== '/') { this.props.history.push(this.state.destinationPath); this.setState({ destinationPath: null }); } From 47266df2401a96935b5edc2f9b67146c32eb15d6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 14:05:53 +0100 Subject: [PATCH 138/206] feat(DB): EE-clickhouse creation scrip multi-run support --- .../db/init_dbs/clickhouse/create/clicks.sql | 38 +- .../db/init_dbs/clickhouse/create/customs.sql | 2 +- .../db/init_dbs/clickhouse/create/errors.sql | 43 +- .../db/init_dbs/clickhouse/create/inputs.sql | 36 +- .../init_dbs/clickhouse/create/longtasks.sql | 46 +- .../clickhouse/create/negatives_buffer.sql | 428 ++++++++--------- .../create/negatives_creation_clickhouse.sql | 434 +++++++++--------- .../db/init_dbs/clickhouse/create/pages.sql | 71 +-- .../clickhouse/create/performance.sql | 59 +-- .../init_dbs/clickhouse/create/resources.sql | 61 +-- .../init_dbs/clickhouse/create/sessions.sql | 2 +- .../clickhouse/create/sessions_metadata.sql | 2 +- 12 files changed, 615 insertions(+), 607 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/clicks.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/clicks.sql index 1fd5b0dbd..7781d2328 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/clicks.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/clicks.sql @@ -1,21 +1,21 @@ -CREATE TABLE clicks +CREATE TABLE IF NOT EXISTS clicks ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - label String, - hesitation_time Nullable(UInt32) + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label String, + hesitation_time Nullable(UInt32) ) ENGINE = MergeTree -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime) -TTL datetime + INTERVAL 1 MONTH; + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime) + TTL datetime + INTERVAL 1 MONTH; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/customs.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/customs.sql index 6d466a7a0..eed67c990 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/customs.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/customs.sql @@ -1,4 +1,4 @@ -CREATE TABLE customs +CREATE TABLE IF NOT EXISTS customs ( session_id UInt64, project_id UInt32, diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/errors.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/errors.sql index 51e2d806a..4560f6500 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/errors.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/errors.sql @@ -1,22 +1,23 @@ -CREATE TABLE errors ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - source Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9), - name Nullable(String), - message String, - error_id String +CREATE TABLE IF NOT EXISTS errors +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + source Enum8('js_exception'=0, 'bugsnag'=1, 'cloudwatch'=2, 'datadog'=3, 'elasticsearch'=4, 'newrelic'=5, 'rollbar'=6, 'sentry'=7, 'stackdriver'=8, 'sumologic'=9), + name Nullable(String), + message String, + error_id String ) ENGINE = MergeTree -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime) -TTL datetime + INTERVAL 1 MONTH; + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime) + TTL datetime + INTERVAL 1 MONTH; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/inputs.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/inputs.sql index 4c369c7d5..523d2d468 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/inputs.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/inputs.sql @@ -1,20 +1,20 @@ -CREATE TABLE inputs +CREATE TABLE IF NOT EXISTS inputs ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - label String + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + label String ) ENGINE = MergeTree -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime) -TTL datetime + INTERVAL 1 MONTH; + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime) + TTL datetime + INTERVAL 1 MONTH; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/longtasks.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/longtasks.sql index a5622407b..9770fb380 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/longtasks.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/longtasks.sql @@ -1,26 +1,26 @@ -CREATE TABLE longtasks +CREATE TABLE IF NOT EXISTS longtasks ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - duration UInt16, - context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8), - container_type Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3), - container_id String, - container_name String, - container_src String + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + duration UInt16, + context Enum8('unknown'=0, 'self'=1, 'same-origin-ancestor'=2, 'same-origin-descendant'=3, 'same-origin'=4, 'cross-origin-ancestor'=5, 'cross-origin-descendant'=6, 'cross-origin-unreachable'=7, 'multiple-contexts'=8), + container_type Enum8('window'=0, 'iframe'=1, 'embed'=2, 'object'=3), + container_id String, + container_name String, + container_src String ) ENGINE = MergeTree -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime) -TTL datetime + INTERVAL 1 MONTH; + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime) + TTL datetime + INTERVAL 1 MONTH; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_buffer.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_buffer.sql index 354e9f87f..ac67028b3 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_buffer.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_buffer.sql @@ -1,213 +1,215 @@ -CREATE TABLE negatives_buffer ( -sessionid UInt64, -clickevent_hesitationtime Nullable(UInt64), -clickevent_label Nullable(String), -clickevent_messageid Nullable(UInt64), -clickevent_timestamp Nullable(Datetime), -connectioninformation_downlink Nullable(UInt64), -connectioninformation_type Nullable(String), -consolelog_level Nullable(String), -consolelog_value Nullable(String), -cpuissue_duration Nullable(UInt64), -cpuissue_rate Nullable(UInt64), -cpuissue_timestamp Nullable(Datetime), -createdocument Nullable(UInt8), -createelementnode_id Nullable(UInt64), -createelementnode_parentid Nullable(UInt64), -cssdeleterule_index Nullable(UInt64), -cssdeleterule_stylesheetid Nullable(UInt64), -cssinsertrule_index Nullable(UInt64), -cssinsertrule_rule Nullable(String), -cssinsertrule_stylesheetid Nullable(UInt64), -customevent_messageid Nullable(UInt64), -customevent_name Nullable(String), -customevent_payload Nullable(String), -customevent_timestamp Nullable(Datetime), -domdrop_timestamp Nullable(Datetime), -errorevent_message Nullable(String), -errorevent_messageid Nullable(UInt64), -errorevent_name Nullable(String), -errorevent_payload Nullable(String), -errorevent_source Nullable(String), -errorevent_timestamp Nullable(Datetime), -fetch_duration Nullable(UInt64), -fetch_method Nullable(String), -fetch_request Nullable(String), -fetch_status Nullable(UInt64), -fetch_timestamp Nullable(Datetime), -fetch_url Nullable(String), -graphql_operationkind Nullable(String), -graphql_operationname Nullable(String), -graphql_response Nullable(String), -graphql_variables Nullable(String), -graphqlevent_messageid Nullable(UInt64), -graphqlevent_name Nullable(String), -graphqlevent_timestamp Nullable(Datetime), -inputevent_label Nullable(String), -inputevent_messageid Nullable(UInt64), -inputevent_timestamp Nullable(Datetime), -inputevent_value Nullable(String), -inputevent_valuemasked Nullable(UInt8), -jsexception_message Nullable(String), -jsexception_name Nullable(String), -jsexception_payload Nullable(String), -longtasks_timestamp Nullable(Datetime), -longtasks_duration Nullable(UInt64), -longtasks_containerid Nullable(String), -longtasks_containersrc Nullable(String), -memoryissue_duration Nullable(UInt64), -memoryissue_rate Nullable(UInt64), -memoryissue_timestamp Nullable(Datetime), -metadata_key Nullable(String), -metadata_value Nullable(String), -mobx_payload Nullable(String), -mobx_type Nullable(String), -mouseclick_id Nullable(UInt64), -mouseclick_hesitationtime Nullable(UInt64), -mouseclick_label Nullable(String), -mousemove_x Nullable(UInt64), -mousemove_y Nullable(UInt64), -movenode_id Nullable(UInt64), -movenode_index Nullable(UInt64), -movenode_parentid Nullable(UInt64), -ngrx_action Nullable(String), -ngrx_duration Nullable(UInt64), -ngrx_state Nullable(String), -pageevent_domcontentloadedeventend Nullable(UInt64), -pageevent_domcontentloadedeventstart Nullable(UInt64), -pageevent_firstcontentfulpaint Nullable(UInt64), -pageevent_firstpaint Nullable(UInt64), -pageevent_loaded Nullable(UInt8), -pageevent_loadeventend Nullable(UInt64), -pageevent_loadeventstart Nullable(UInt64), -pageevent_messageid Nullable(UInt64), -pageevent_referrer Nullable(String), -pageevent_requeststart Nullable(UInt64), -pageevent_responseend Nullable(UInt64), -pageevent_responsestart Nullable(UInt64), -pageevent_speedindex Nullable(UInt64), -pageevent_timestamp Nullable(Datetime), -pageevent_url Nullable(String), -pageloadtiming_domcontentloadedeventend Nullable(UInt64), -pageloadtiming_domcontentloadedeventstart Nullable(UInt64), -pageloadtiming_firstcontentfulpaint Nullable(UInt64), -pageloadtiming_firstpaint Nullable(UInt64), -pageloadtiming_loadeventend Nullable(UInt64), -pageloadtiming_loadeventstart Nullable(UInt64), -pageloadtiming_requeststart Nullable(UInt64), -pageloadtiming_responseend Nullable(UInt64), -pageloadtiming_responsestart Nullable(UInt64), -pagerendertiming_speedindex Nullable(UInt64), -pagerendertiming_timetointeractive Nullable(UInt64), -pagerendertiming_visuallycomplete Nullable(UInt64), -performancetrack_frames Nullable(Int64), -performancetrack_ticks Nullable(Int64), -performancetrack_totaljsheapsize Nullable(UInt64), -performancetrack_usedjsheapsize Nullable(UInt64), -performancetrackaggr_avgcpu Nullable(UInt64), -performancetrackaggr_avgfps Nullable(UInt64), -performancetrackaggr_avgtotaljsheapsize Nullable(UInt64), -performancetrackaggr_avgusedjsheapsize Nullable(UInt64), -performancetrackaggr_maxcpu Nullable(UInt64), -performancetrackaggr_maxfps Nullable(UInt64), -performancetrackaggr_maxtotaljsheapsize Nullable(UInt64), -performancetrackaggr_maxusedjsheapsize Nullable(UInt64), -performancetrackaggr_mincpu Nullable(UInt64), -performancetrackaggr_minfps Nullable(UInt64), -performancetrackaggr_mintotaljsheapsize Nullable(UInt64), -performancetrackaggr_minusedjsheapsize Nullable(UInt64), -performancetrackaggr_timestampend Nullable(Datetime), -performancetrackaggr_timestampstart Nullable(Datetime), -profiler_args Nullable(String), -profiler_duration Nullable(UInt64), -profiler_name Nullable(String), -profiler_result Nullable(String), -rawcustomevent_name Nullable(String), -rawcustomevent_payload Nullable(String), -rawerrorevent_message Nullable(String), -rawerrorevent_name Nullable(String), -rawerrorevent_payload Nullable(String), -rawerrorevent_source Nullable(String), -rawerrorevent_timestamp Nullable(Datetime), -redux_action Nullable(String), -redux_duration Nullable(UInt64), -redux_state Nullable(String), -removenode_id Nullable(UInt64), -removenodeattribute_id Nullable(UInt64), -removenodeattribute_name Nullable(String), -resourceevent_decodedbodysize Nullable(UInt64), -resourceevent_duration Nullable(UInt64), -resourceevent_encodedbodysize Nullable(UInt64), -resourceevent_headersize Nullable(UInt64), -resourceevent_messageid Nullable(UInt64), -resourceevent_method Nullable(String), -resourceevent_status Nullable(UInt64), -resourceevent_success Nullable(UInt8), -resourceevent_timestamp Nullable(Datetime), -resourceevent_ttfb Nullable(UInt64), -resourceevent_type Nullable(String), -resourceevent_url Nullable(String), -resourcetiming_decodedbodysize Nullable(UInt64), -resourcetiming_duration Nullable(UInt64), -resourcetiming_encodedbodysize Nullable(UInt64), -resourcetiming_headersize Nullable(UInt64), -resourcetiming_initiator Nullable(String), -resourcetiming_timestamp Nullable(Datetime), -resourcetiming_ttfb Nullable(UInt64), -resourcetiming_url Nullable(String), -sessiondisconnect Nullable(UInt8), -sessiondisconnect_timestamp Nullable(Datetime), -sessionend Nullable(UInt8), -sessionend_timestamp Nullable(Datetime), -sessionstart_projectid Nullable(UInt64), -sessionstart_revid Nullable(String), -sessionstart_timestamp Nullable(Datetime), -sessionstart_trackerversion Nullable(String), -sessionstart_useragent Nullable(String), -sessionstart_userbrowser Nullable(String), -sessionstart_userbrowserversion Nullable(String), -sessionstart_usercountry Nullable(String), -sessionstart_userdevice Nullable(String), -sessionstart_userdeviceheapsize Nullable(UInt64), -sessionstart_userdevicememorysize Nullable(UInt64), -sessionstart_userdevicetype Nullable(String), -sessionstart_useros Nullable(String), -sessionstart_userosversion Nullable(String), -sessionstart_useruuid Nullable(String), -setcssdata_data Nullable(UInt64), -setcssdata_id Nullable(UInt64), -setinputchecked_checked Nullable(UInt64), -setinputchecked_id Nullable(UInt64), -setinputtarget_id Nullable(UInt64), -setinputtarget_label Nullable(UInt64), -setinputvalue_id Nullable(UInt64), -setinputvalue_mask Nullable(UInt64), -setinputvalue_value Nullable(UInt64), -setnodeattribute_id Nullable(UInt64), -setnodeattribute_name Nullable(UInt64), -setnodeattribute_value Nullable(UInt64), -setnodedata_data Nullable(UInt64), -setnodedata_id Nullable(UInt64), -setnodescroll_id Nullable(UInt64), -setnodescroll_x Nullable(Int64), -setnodescroll_y Nullable(Int64), -setpagelocation_navigationstart Nullable(UInt64), -setpagelocation_referrer Nullable(String), -setpagelocation_url Nullable(String), -setpagevisibility_hidden Nullable(UInt8), -setviewportscroll_x Nullable(Int64), -setviewportscroll_y Nullable(Int64), -setviewportsize_height Nullable(UInt64), -setviewportsize_width Nullable(UInt64), -stateaction_type Nullable(String), -stateactionevent_messageid Nullable(UInt64), -stateactionevent_timestamp Nullable(Datetime), -stateactionevent_type Nullable(String), -timestamp_timestamp Nullable(Datetime), -useranonymousid_id Nullable(String), -userid_id Nullable(String), -vuex_mutation Nullable(String), -vuex_state Nullable(String), -received_at Datetime, -batch_order_number Int64) -ENGINE = Buffer(default, negatives, 16, 10, 120, 10000, 1000000, 10000, 100000000); +CREATE TABLE IF NOT EXISTS negatives_buffer +( + sessionid UInt64, + clickevent_hesitationtime Nullable(UInt64), + clickevent_label Nullable(String), + clickevent_messageid Nullable(UInt64), + clickevent_timestamp Nullable(Datetime), + connectioninformation_downlink Nullable(UInt64), + connectioninformation_type Nullable(String), + consolelog_level Nullable(String), + consolelog_value Nullable(String), + cpuissue_duration Nullable(UInt64), + cpuissue_rate Nullable(UInt64), + cpuissue_timestamp Nullable(Datetime), + createdocument Nullable(UInt8), + createelementnode_id Nullable(UInt64), + createelementnode_parentid Nullable(UInt64), + cssdeleterule_index Nullable(UInt64), + cssdeleterule_stylesheetid Nullable(UInt64), + cssinsertrule_index Nullable(UInt64), + cssinsertrule_rule Nullable(String), + cssinsertrule_stylesheetid Nullable(UInt64), + customevent_messageid Nullable(UInt64), + customevent_name Nullable(String), + customevent_payload Nullable(String), + customevent_timestamp Nullable(Datetime), + domdrop_timestamp Nullable(Datetime), + errorevent_message Nullable(String), + errorevent_messageid Nullable(UInt64), + errorevent_name Nullable(String), + errorevent_payload Nullable(String), + errorevent_source Nullable(String), + errorevent_timestamp Nullable(Datetime), + fetch_duration Nullable(UInt64), + fetch_method Nullable(String), + fetch_request Nullable(String), + fetch_status Nullable(UInt64), + fetch_timestamp Nullable(Datetime), + fetch_url Nullable(String), + graphql_operationkind Nullable(String), + graphql_operationname Nullable(String), + graphql_response Nullable(String), + graphql_variables Nullable(String), + graphqlevent_messageid Nullable(UInt64), + graphqlevent_name Nullable(String), + graphqlevent_timestamp Nullable(Datetime), + inputevent_label Nullable(String), + inputevent_messageid Nullable(UInt64), + inputevent_timestamp Nullable(Datetime), + inputevent_value Nullable(String), + inputevent_valuemasked Nullable(UInt8), + jsexception_message Nullable(String), + jsexception_name Nullable(String), + jsexception_payload Nullable(String), + longtasks_timestamp Nullable(Datetime), + longtasks_duration Nullable(UInt64), + longtasks_containerid Nullable(String), + longtasks_containersrc Nullable(String), + memoryissue_duration Nullable(UInt64), + memoryissue_rate Nullable(UInt64), + memoryissue_timestamp Nullable(Datetime), + metadata_key Nullable(String), + metadata_value Nullable(String), + mobx_payload Nullable(String), + mobx_type Nullable(String), + mouseclick_id Nullable(UInt64), + mouseclick_hesitationtime Nullable(UInt64), + mouseclick_label Nullable(String), + mousemove_x Nullable(UInt64), + mousemove_y Nullable(UInt64), + movenode_id Nullable(UInt64), + movenode_index Nullable(UInt64), + movenode_parentid Nullable(UInt64), + ngrx_action Nullable(String), + ngrx_duration Nullable(UInt64), + ngrx_state Nullable(String), + pageevent_domcontentloadedeventend Nullable(UInt64), + pageevent_domcontentloadedeventstart Nullable(UInt64), + pageevent_firstcontentfulpaint Nullable(UInt64), + pageevent_firstpaint Nullable(UInt64), + pageevent_loaded Nullable(UInt8), + pageevent_loadeventend Nullable(UInt64), + pageevent_loadeventstart Nullable(UInt64), + pageevent_messageid Nullable(UInt64), + pageevent_referrer Nullable(String), + pageevent_requeststart Nullable(UInt64), + pageevent_responseend Nullable(UInt64), + pageevent_responsestart Nullable(UInt64), + pageevent_speedindex Nullable(UInt64), + pageevent_timestamp Nullable(Datetime), + pageevent_url Nullable(String), + pageloadtiming_domcontentloadedeventend Nullable(UInt64), + pageloadtiming_domcontentloadedeventstart Nullable(UInt64), + pageloadtiming_firstcontentfulpaint Nullable(UInt64), + pageloadtiming_firstpaint Nullable(UInt64), + pageloadtiming_loadeventend Nullable(UInt64), + pageloadtiming_loadeventstart Nullable(UInt64), + pageloadtiming_requeststart Nullable(UInt64), + pageloadtiming_responseend Nullable(UInt64), + pageloadtiming_responsestart Nullable(UInt64), + pagerendertiming_speedindex Nullable(UInt64), + pagerendertiming_timetointeractive Nullable(UInt64), + pagerendertiming_visuallycomplete Nullable(UInt64), + performancetrack_frames Nullable(Int64), + performancetrack_ticks Nullable(Int64), + performancetrack_totaljsheapsize Nullable(UInt64), + performancetrack_usedjsheapsize Nullable(UInt64), + performancetrackaggr_avgcpu Nullable(UInt64), + performancetrackaggr_avgfps Nullable(UInt64), + performancetrackaggr_avgtotaljsheapsize Nullable(UInt64), + performancetrackaggr_avgusedjsheapsize Nullable(UInt64), + performancetrackaggr_maxcpu Nullable(UInt64), + performancetrackaggr_maxfps Nullable(UInt64), + performancetrackaggr_maxtotaljsheapsize Nullable(UInt64), + performancetrackaggr_maxusedjsheapsize Nullable(UInt64), + performancetrackaggr_mincpu Nullable(UInt64), + performancetrackaggr_minfps Nullable(UInt64), + performancetrackaggr_mintotaljsheapsize Nullable(UInt64), + performancetrackaggr_minusedjsheapsize Nullable(UInt64), + performancetrackaggr_timestampend Nullable(Datetime), + performancetrackaggr_timestampstart Nullable(Datetime), + profiler_args Nullable(String), + profiler_duration Nullable(UInt64), + profiler_name Nullable(String), + profiler_result Nullable(String), + rawcustomevent_name Nullable(String), + rawcustomevent_payload Nullable(String), + rawerrorevent_message Nullable(String), + rawerrorevent_name Nullable(String), + rawerrorevent_payload Nullable(String), + rawerrorevent_source Nullable(String), + rawerrorevent_timestamp Nullable(Datetime), + redux_action Nullable(String), + redux_duration Nullable(UInt64), + redux_state Nullable(String), + removenode_id Nullable(UInt64), + removenodeattribute_id Nullable(UInt64), + removenodeattribute_name Nullable(String), + resourceevent_decodedbodysize Nullable(UInt64), + resourceevent_duration Nullable(UInt64), + resourceevent_encodedbodysize Nullable(UInt64), + resourceevent_headersize Nullable(UInt64), + resourceevent_messageid Nullable(UInt64), + resourceevent_method Nullable(String), + resourceevent_status Nullable(UInt64), + resourceevent_success Nullable(UInt8), + resourceevent_timestamp Nullable(Datetime), + resourceevent_ttfb Nullable(UInt64), + resourceevent_type Nullable(String), + resourceevent_url Nullable(String), + resourcetiming_decodedbodysize Nullable(UInt64), + resourcetiming_duration Nullable(UInt64), + resourcetiming_encodedbodysize Nullable(UInt64), + resourcetiming_headersize Nullable(UInt64), + resourcetiming_initiator Nullable(String), + resourcetiming_timestamp Nullable(Datetime), + resourcetiming_ttfb Nullable(UInt64), + resourcetiming_url Nullable(String), + sessiondisconnect Nullable(UInt8), + sessiondisconnect_timestamp Nullable(Datetime), + sessionend Nullable(UInt8), + sessionend_timestamp Nullable(Datetime), + sessionstart_projectid Nullable(UInt64), + sessionstart_revid Nullable(String), + sessionstart_timestamp Nullable(Datetime), + sessionstart_trackerversion Nullable(String), + sessionstart_useragent Nullable(String), + sessionstart_userbrowser Nullable(String), + sessionstart_userbrowserversion Nullable(String), + sessionstart_usercountry Nullable(String), + sessionstart_userdevice Nullable(String), + sessionstart_userdeviceheapsize Nullable(UInt64), + sessionstart_userdevicememorysize Nullable(UInt64), + sessionstart_userdevicetype Nullable(String), + sessionstart_useros Nullable(String), + sessionstart_userosversion Nullable(String), + sessionstart_useruuid Nullable(String), + setcssdata_data Nullable(UInt64), + setcssdata_id Nullable(UInt64), + setinputchecked_checked Nullable(UInt64), + setinputchecked_id Nullable(UInt64), + setinputtarget_id Nullable(UInt64), + setinputtarget_label Nullable(UInt64), + setinputvalue_id Nullable(UInt64), + setinputvalue_mask Nullable(UInt64), + setinputvalue_value Nullable(UInt64), + setnodeattribute_id Nullable(UInt64), + setnodeattribute_name Nullable(UInt64), + setnodeattribute_value Nullable(UInt64), + setnodedata_data Nullable(UInt64), + setnodedata_id Nullable(UInt64), + setnodescroll_id Nullable(UInt64), + setnodescroll_x Nullable(Int64), + setnodescroll_y Nullable(Int64), + setpagelocation_navigationstart Nullable(UInt64), + setpagelocation_referrer Nullable(String), + setpagelocation_url Nullable(String), + setpagevisibility_hidden Nullable(UInt8), + setviewportscroll_x Nullable(Int64), + setviewportscroll_y Nullable(Int64), + setviewportsize_height Nullable(UInt64), + setviewportsize_width Nullable(UInt64), + stateaction_type Nullable(String), + stateactionevent_messageid Nullable(UInt64), + stateactionevent_timestamp Nullable(Datetime), + stateactionevent_type Nullable(String), + timestamp_timestamp Nullable(Datetime), + useranonymousid_id Nullable(String), + userid_id Nullable(String), + vuex_mutation Nullable(String), + vuex_state Nullable(String), + received_at Datetime, + batch_order_number Int64 +) + ENGINE = Buffer(default, negatives, 16, 10, 120, 10000, 1000000, 10000, 100000000); diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_creation_clickhouse.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_creation_clickhouse.sql index 7b69a63cb..361082d7c 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_creation_clickhouse.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/negatives_creation_clickhouse.sql @@ -1,216 +1,218 @@ -create table negatives ( -sessionid UInt64, -clickevent_hesitationtime Nullable(UInt64), -clickevent_label Nullable(String), -clickevent_messageid Nullable(UInt64), -clickevent_timestamp Nullable(Datetime), -connectioninformation_downlink Nullable(UInt64), -connectioninformation_type Nullable(String), -consolelog_level Nullable(String), -consolelog_value Nullable(String), -cpuissue_duration Nullable(UInt64), -cpuissue_rate Nullable(UInt64), -cpuissue_timestamp Nullable(Datetime), -createdocument Nullable(UInt8), -createelementnode_id Nullable(UInt64), -createelementnode_parentid Nullable(UInt64), -cssdeleterule_index Nullable(UInt64), -cssdeleterule_stylesheetid Nullable(UInt64), -cssinsertrule_index Nullable(UInt64), -cssinsertrule_rule Nullable(String), -cssinsertrule_stylesheetid Nullable(UInt64), -customevent_messageid Nullable(UInt64), -customevent_name Nullable(String), -customevent_payload Nullable(String), -customevent_timestamp Nullable(Datetime), -domdrop_timestamp Nullable(Datetime), -errorevent_message Nullable(String), -errorevent_messageid Nullable(UInt64), -errorevent_name Nullable(String), -errorevent_payload Nullable(String), -errorevent_source Nullable(String), -errorevent_timestamp Nullable(Datetime), -fetch_duration Nullable(UInt64), -fetch_method Nullable(String), -fetch_request Nullable(String), -fetch_status Nullable(UInt64), -fetch_timestamp Nullable(Datetime), -fetch_url Nullable(String), -graphql_operationkind Nullable(String), -graphql_operationname Nullable(String), -graphql_response Nullable(String), -graphql_variables Nullable(String), -graphqlevent_messageid Nullable(UInt64), -graphqlevent_name Nullable(String), -graphqlevent_timestamp Nullable(Datetime), -inputevent_label Nullable(String), -inputevent_messageid Nullable(UInt64), -inputevent_timestamp Nullable(Datetime), -inputevent_value Nullable(String), -inputevent_valuemasked Nullable(UInt8), -jsexception_message Nullable(String), -jsexception_name Nullable(String), -jsexception_payload Nullable(String), -longtasks_timestamp Nullable(Datetime), -longtasks_duration Nullable(UInt64), -longtasks_containerid Nullable(String), -longtasks_containersrc Nullable(String), -memoryissue_duration Nullable(UInt64), -memoryissue_rate Nullable(UInt64), -memoryissue_timestamp Nullable(Datetime), -metadata_key Nullable(String), -metadata_value Nullable(String), -mobx_payload Nullable(String), -mobx_type Nullable(String), -mouseclick_id Nullable(UInt64), -mouseclick_hesitationtime Nullable(UInt64), -mouseclick_label Nullable(String), -mousemove_x Nullable(UInt64), -mousemove_y Nullable(UInt64), -movenode_id Nullable(UInt64), -movenode_index Nullable(UInt64), -movenode_parentid Nullable(UInt64), -ngrx_action Nullable(String), -ngrx_duration Nullable(UInt64), -ngrx_state Nullable(String), -pageevent_domcontentloadedeventend Nullable(UInt64), -pageevent_domcontentloadedeventstart Nullable(UInt64), -pageevent_firstcontentfulpaint Nullable(UInt64), -pageevent_firstpaint Nullable(UInt64), -pageevent_loaded Nullable(UInt8), -pageevent_loadeventend Nullable(UInt64), -pageevent_loadeventstart Nullable(UInt64), -pageevent_messageid Nullable(UInt64), -pageevent_referrer Nullable(String), -pageevent_requeststart Nullable(UInt64), -pageevent_responseend Nullable(UInt64), -pageevent_responsestart Nullable(UInt64), -pageevent_speedindex Nullable(UInt64), -pageevent_timestamp Nullable(Datetime), -pageevent_url Nullable(String), -pageloadtiming_domcontentloadedeventend Nullable(UInt64), -pageloadtiming_domcontentloadedeventstart Nullable(UInt64), -pageloadtiming_firstcontentfulpaint Nullable(UInt64), -pageloadtiming_firstpaint Nullable(UInt64), -pageloadtiming_loadeventend Nullable(UInt64), -pageloadtiming_loadeventstart Nullable(UInt64), -pageloadtiming_requeststart Nullable(UInt64), -pageloadtiming_responseend Nullable(UInt64), -pageloadtiming_responsestart Nullable(UInt64), -pagerendertiming_speedindex Nullable(UInt64), -pagerendertiming_timetointeractive Nullable(UInt64), -pagerendertiming_visuallycomplete Nullable(UInt64), -performancetrack_frames Nullable(Int64), -performancetrack_ticks Nullable(Int64), -performancetrack_totaljsheapsize Nullable(UInt64), -performancetrack_usedjsheapsize Nullable(UInt64), -performancetrackaggr_avgcpu Nullable(UInt64), -performancetrackaggr_avgfps Nullable(UInt64), -performancetrackaggr_avgtotaljsheapsize Nullable(UInt64), -performancetrackaggr_avgusedjsheapsize Nullable(UInt64), -performancetrackaggr_maxcpu Nullable(UInt64), -performancetrackaggr_maxfps Nullable(UInt64), -performancetrackaggr_maxtotaljsheapsize Nullable(UInt64), -performancetrackaggr_maxusedjsheapsize Nullable(UInt64), -performancetrackaggr_mincpu Nullable(UInt64), -performancetrackaggr_minfps Nullable(UInt64), -performancetrackaggr_mintotaljsheapsize Nullable(UInt64), -performancetrackaggr_minusedjsheapsize Nullable(UInt64), -performancetrackaggr_timestampend Nullable(Datetime), -performancetrackaggr_timestampstart Nullable(Datetime), -profiler_args Nullable(String), -profiler_duration Nullable(UInt64), -profiler_name Nullable(String), -profiler_result Nullable(String), -rawcustomevent_name Nullable(String), -rawcustomevent_payload Nullable(String), -rawerrorevent_message Nullable(String), -rawerrorevent_name Nullable(String), -rawerrorevent_payload Nullable(String), -rawerrorevent_source Nullable(String), -rawerrorevent_timestamp Nullable(Datetime), -redux_action Nullable(String), -redux_duration Nullable(UInt64), -redux_state Nullable(String), -removenode_id Nullable(UInt64), -removenodeattribute_id Nullable(UInt64), -removenodeattribute_name Nullable(String), -resourceevent_decodedbodysize Nullable(UInt64), -resourceevent_duration Nullable(UInt64), -resourceevent_encodedbodysize Nullable(UInt64), -resourceevent_headersize Nullable(UInt64), -resourceevent_messageid Nullable(UInt64), -resourceevent_method Nullable(String), -resourceevent_status Nullable(UInt64), -resourceevent_success Nullable(UInt8), -resourceevent_timestamp Nullable(Datetime), -resourceevent_ttfb Nullable(UInt64), -resourceevent_type Nullable(String), -resourceevent_url Nullable(String), -resourcetiming_decodedbodysize Nullable(UInt64), -resourcetiming_duration Nullable(UInt64), -resourcetiming_encodedbodysize Nullable(UInt64), -resourcetiming_headersize Nullable(UInt64), -resourcetiming_initiator Nullable(String), -resourcetiming_timestamp Nullable(Datetime), -resourcetiming_ttfb Nullable(UInt64), -resourcetiming_url Nullable(String), -sessiondisconnect Nullable(UInt8), -sessiondisconnect_timestamp Nullable(Datetime), -sessionend Nullable(UInt8), -sessionend_timestamp Nullable(Datetime), -sessionstart_projectid Nullable(UInt64), -sessionstart_revid Nullable(String), -sessionstart_timestamp Nullable(Datetime), -sessionstart_trackerversion Nullable(String), -sessionstart_useragent Nullable(String), -sessionstart_userbrowser Nullable(String), -sessionstart_userbrowserversion Nullable(String), -sessionstart_usercountry Nullable(String), -sessionstart_userdevice Nullable(String), -sessionstart_userdeviceheapsize Nullable(UInt64), -sessionstart_userdevicememorysize Nullable(UInt64), -sessionstart_userdevicetype Nullable(String), -sessionstart_useros Nullable(String), -sessionstart_userosversion Nullable(String), -sessionstart_useruuid Nullable(String), -setcssdata_data Nullable(UInt64), -setcssdata_id Nullable(UInt64), -setinputchecked_checked Nullable(UInt64), -setinputchecked_id Nullable(UInt64), -setinputtarget_id Nullable(UInt64), -setinputtarget_label Nullable(UInt64), -setinputvalue_id Nullable(UInt64), -setinputvalue_mask Nullable(UInt64), -setinputvalue_value Nullable(UInt64), -setnodeattribute_id Nullable(UInt64), -setnodeattribute_name Nullable(UInt64), -setnodeattribute_value Nullable(UInt64), -setnodedata_data Nullable(UInt64), -setnodedata_id Nullable(UInt64), -setnodescroll_id Nullable(UInt64), -setnodescroll_x Nullable(Int64), -setnodescroll_y Nullable(Int64), -setpagelocation_navigationstart Nullable(UInt64), -setpagelocation_referrer Nullable(String), -setpagelocation_url Nullable(String), -setpagevisibility_hidden Nullable(UInt8), -setviewportscroll_x Nullable(Int64), -setviewportscroll_y Nullable(Int64), -setviewportsize_height Nullable(UInt64), -setviewportsize_width Nullable(UInt64), -stateaction_type Nullable(String), -stateactionevent_messageid Nullable(UInt64), -stateactionevent_timestamp Nullable(Datetime), -stateactionevent_type Nullable(String), -timestamp_timestamp Nullable(Datetime), -useranonymousid_id Nullable(String), -userid_id Nullable(String), -vuex_mutation Nullable(String), -vuex_state Nullable(String), -received_at Datetime, -batch_order_number Int64) -ENGINE = MergeTree() -PARTITION BY toYYYYMM(received_at) -ORDER BY (received_at, batch_order_number) -SETTINGS min_bytes_for_wide_part=1, use_minimalistic_part_header_in_zookeeper=1; \ No newline at end of file +CREATE TABLE IF NOT EXISTS negatives +( + sessionid UInt64, + clickevent_hesitationtime Nullable(UInt64), + clickevent_label Nullable(String), + clickevent_messageid Nullable(UInt64), + clickevent_timestamp Nullable(Datetime), + connectioninformation_downlink Nullable(UInt64), + connectioninformation_type Nullable(String), + consolelog_level Nullable(String), + consolelog_value Nullable(String), + cpuissue_duration Nullable(UInt64), + cpuissue_rate Nullable(UInt64), + cpuissue_timestamp Nullable(Datetime), + createdocument Nullable(UInt8), + createelementnode_id Nullable(UInt64), + createelementnode_parentid Nullable(UInt64), + cssdeleterule_index Nullable(UInt64), + cssdeleterule_stylesheetid Nullable(UInt64), + cssinsertrule_index Nullable(UInt64), + cssinsertrule_rule Nullable(String), + cssinsertrule_stylesheetid Nullable(UInt64), + customevent_messageid Nullable(UInt64), + customevent_name Nullable(String), + customevent_payload Nullable(String), + customevent_timestamp Nullable(Datetime), + domdrop_timestamp Nullable(Datetime), + errorevent_message Nullable(String), + errorevent_messageid Nullable(UInt64), + errorevent_name Nullable(String), + errorevent_payload Nullable(String), + errorevent_source Nullable(String), + errorevent_timestamp Nullable(Datetime), + fetch_duration Nullable(UInt64), + fetch_method Nullable(String), + fetch_request Nullable(String), + fetch_status Nullable(UInt64), + fetch_timestamp Nullable(Datetime), + fetch_url Nullable(String), + graphql_operationkind Nullable(String), + graphql_operationname Nullable(String), + graphql_response Nullable(String), + graphql_variables Nullable(String), + graphqlevent_messageid Nullable(UInt64), + graphqlevent_name Nullable(String), + graphqlevent_timestamp Nullable(Datetime), + inputevent_label Nullable(String), + inputevent_messageid Nullable(UInt64), + inputevent_timestamp Nullable(Datetime), + inputevent_value Nullable(String), + inputevent_valuemasked Nullable(UInt8), + jsexception_message Nullable(String), + jsexception_name Nullable(String), + jsexception_payload Nullable(String), + longtasks_timestamp Nullable(Datetime), + longtasks_duration Nullable(UInt64), + longtasks_containerid Nullable(String), + longtasks_containersrc Nullable(String), + memoryissue_duration Nullable(UInt64), + memoryissue_rate Nullable(UInt64), + memoryissue_timestamp Nullable(Datetime), + metadata_key Nullable(String), + metadata_value Nullable(String), + mobx_payload Nullable(String), + mobx_type Nullable(String), + mouseclick_id Nullable(UInt64), + mouseclick_hesitationtime Nullable(UInt64), + mouseclick_label Nullable(String), + mousemove_x Nullable(UInt64), + mousemove_y Nullable(UInt64), + movenode_id Nullable(UInt64), + movenode_index Nullable(UInt64), + movenode_parentid Nullable(UInt64), + ngrx_action Nullable(String), + ngrx_duration Nullable(UInt64), + ngrx_state Nullable(String), + pageevent_domcontentloadedeventend Nullable(UInt64), + pageevent_domcontentloadedeventstart Nullable(UInt64), + pageevent_firstcontentfulpaint Nullable(UInt64), + pageevent_firstpaint Nullable(UInt64), + pageevent_loaded Nullable(UInt8), + pageevent_loadeventend Nullable(UInt64), + pageevent_loadeventstart Nullable(UInt64), + pageevent_messageid Nullable(UInt64), + pageevent_referrer Nullable(String), + pageevent_requeststart Nullable(UInt64), + pageevent_responseend Nullable(UInt64), + pageevent_responsestart Nullable(UInt64), + pageevent_speedindex Nullable(UInt64), + pageevent_timestamp Nullable(Datetime), + pageevent_url Nullable(String), + pageloadtiming_domcontentloadedeventend Nullable(UInt64), + pageloadtiming_domcontentloadedeventstart Nullable(UInt64), + pageloadtiming_firstcontentfulpaint Nullable(UInt64), + pageloadtiming_firstpaint Nullable(UInt64), + pageloadtiming_loadeventend Nullable(UInt64), + pageloadtiming_loadeventstart Nullable(UInt64), + pageloadtiming_requeststart Nullable(UInt64), + pageloadtiming_responseend Nullable(UInt64), + pageloadtiming_responsestart Nullable(UInt64), + pagerendertiming_speedindex Nullable(UInt64), + pagerendertiming_timetointeractive Nullable(UInt64), + pagerendertiming_visuallycomplete Nullable(UInt64), + performancetrack_frames Nullable(Int64), + performancetrack_ticks Nullable(Int64), + performancetrack_totaljsheapsize Nullable(UInt64), + performancetrack_usedjsheapsize Nullable(UInt64), + performancetrackaggr_avgcpu Nullable(UInt64), + performancetrackaggr_avgfps Nullable(UInt64), + performancetrackaggr_avgtotaljsheapsize Nullable(UInt64), + performancetrackaggr_avgusedjsheapsize Nullable(UInt64), + performancetrackaggr_maxcpu Nullable(UInt64), + performancetrackaggr_maxfps Nullable(UInt64), + performancetrackaggr_maxtotaljsheapsize Nullable(UInt64), + performancetrackaggr_maxusedjsheapsize Nullable(UInt64), + performancetrackaggr_mincpu Nullable(UInt64), + performancetrackaggr_minfps Nullable(UInt64), + performancetrackaggr_mintotaljsheapsize Nullable(UInt64), + performancetrackaggr_minusedjsheapsize Nullable(UInt64), + performancetrackaggr_timestampend Nullable(Datetime), + performancetrackaggr_timestampstart Nullable(Datetime), + profiler_args Nullable(String), + profiler_duration Nullable(UInt64), + profiler_name Nullable(String), + profiler_result Nullable(String), + rawcustomevent_name Nullable(String), + rawcustomevent_payload Nullable(String), + rawerrorevent_message Nullable(String), + rawerrorevent_name Nullable(String), + rawerrorevent_payload Nullable(String), + rawerrorevent_source Nullable(String), + rawerrorevent_timestamp Nullable(Datetime), + redux_action Nullable(String), + redux_duration Nullable(UInt64), + redux_state Nullable(String), + removenode_id Nullable(UInt64), + removenodeattribute_id Nullable(UInt64), + removenodeattribute_name Nullable(String), + resourceevent_decodedbodysize Nullable(UInt64), + resourceevent_duration Nullable(UInt64), + resourceevent_encodedbodysize Nullable(UInt64), + resourceevent_headersize Nullable(UInt64), + resourceevent_messageid Nullable(UInt64), + resourceevent_method Nullable(String), + resourceevent_status Nullable(UInt64), + resourceevent_success Nullable(UInt8), + resourceevent_timestamp Nullable(Datetime), + resourceevent_ttfb Nullable(UInt64), + resourceevent_type Nullable(String), + resourceevent_url Nullable(String), + resourcetiming_decodedbodysize Nullable(UInt64), + resourcetiming_duration Nullable(UInt64), + resourcetiming_encodedbodysize Nullable(UInt64), + resourcetiming_headersize Nullable(UInt64), + resourcetiming_initiator Nullable(String), + resourcetiming_timestamp Nullable(Datetime), + resourcetiming_ttfb Nullable(UInt64), + resourcetiming_url Nullable(String), + sessiondisconnect Nullable(UInt8), + sessiondisconnect_timestamp Nullable(Datetime), + sessionend Nullable(UInt8), + sessionend_timestamp Nullable(Datetime), + sessionstart_projectid Nullable(UInt64), + sessionstart_revid Nullable(String), + sessionstart_timestamp Nullable(Datetime), + sessionstart_trackerversion Nullable(String), + sessionstart_useragent Nullable(String), + sessionstart_userbrowser Nullable(String), + sessionstart_userbrowserversion Nullable(String), + sessionstart_usercountry Nullable(String), + sessionstart_userdevice Nullable(String), + sessionstart_userdeviceheapsize Nullable(UInt64), + sessionstart_userdevicememorysize Nullable(UInt64), + sessionstart_userdevicetype Nullable(String), + sessionstart_useros Nullable(String), + sessionstart_userosversion Nullable(String), + sessionstart_useruuid Nullable(String), + setcssdata_data Nullable(UInt64), + setcssdata_id Nullable(UInt64), + setinputchecked_checked Nullable(UInt64), + setinputchecked_id Nullable(UInt64), + setinputtarget_id Nullable(UInt64), + setinputtarget_label Nullable(UInt64), + setinputvalue_id Nullable(UInt64), + setinputvalue_mask Nullable(UInt64), + setinputvalue_value Nullable(UInt64), + setnodeattribute_id Nullable(UInt64), + setnodeattribute_name Nullable(UInt64), + setnodeattribute_value Nullable(UInt64), + setnodedata_data Nullable(UInt64), + setnodedata_id Nullable(UInt64), + setnodescroll_id Nullable(UInt64), + setnodescroll_x Nullable(Int64), + setnodescroll_y Nullable(Int64), + setpagelocation_navigationstart Nullable(UInt64), + setpagelocation_referrer Nullable(String), + setpagelocation_url Nullable(String), + setpagevisibility_hidden Nullable(UInt8), + setviewportscroll_x Nullable(Int64), + setviewportscroll_y Nullable(Int64), + setviewportsize_height Nullable(UInt64), + setviewportsize_width Nullable(UInt64), + stateaction_type Nullable(String), + stateactionevent_messageid Nullable(UInt64), + stateactionevent_timestamp Nullable(Datetime), + stateactionevent_type Nullable(String), + timestamp_timestamp Nullable(Datetime), + useranonymousid_id Nullable(String), + userid_id Nullable(String), + vuex_mutation Nullable(String), + vuex_state Nullable(String), + received_at Datetime, + batch_order_number Int64 +) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(received_at) + ORDER BY (received_at, batch_order_number) + SETTINGS min_bytes_for_wide_part = 1, use_minimalistic_part_header_in_zookeeper = 1; \ No newline at end of file diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/pages.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/pages.sql index 832f3423d..71d9503cf 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/pages.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/pages.sql @@ -1,38 +1,39 @@ -CREATE TABLE pages ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - url String, - url_host String MATERIALIZED lower(domain(url)), - url_path String MATERIALIZED lower(pathFull(url)), - request_start Nullable(UInt16), - response_start Nullable(UInt16), - response_end Nullable(UInt16), - dom_content_loaded_event_start Nullable(UInt16), - dom_content_loaded_event_end Nullable(UInt16), - load_event_start Nullable(UInt16), - load_event_end Nullable(UInt16), - first_paint Nullable(UInt16), - first_contentful_paint Nullable(UInt16), - speed_index Nullable(UInt16), - visually_complete Nullable(UInt16), - time_to_interactive Nullable(UInt16), - ttfb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_start, request_start), minus(response_start, request_start), Null), - ttlb Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, request_start), minus(response_end, request_start), Null), - response_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(response_end, response_start), minus(response_end, response_start), Null), - dom_building_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(dom_content_loaded_event_start, response_end), minus(dom_content_loaded_event_start, response_end), Null), - dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), - load_event_time Nullable(UInt16) MATERIALIZED if(greaterOrEquals(load_event_end, load_event_start), minus(load_event_end, load_event_start), Null) +CREATE TABLE IF NOT EXISTS pages +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + url String, + url_host String MATERIALIZED lower(domain (url)), + url_path String MATERIALIZED lower(pathFull(url)), + request_start Nullable(UInt16), + response_start Nullable(UInt16), + response_end Nullable(UInt16), + dom_content_loaded_event_start Nullable(UInt16), + dom_content_loaded_event_end Nullable(UInt16), + load_event_start Nullable(UInt16), + load_event_end Nullable(UInt16), + first_paint Nullable(UInt16), + first_contentful_paint Nullable(UInt16), + speed_index Nullable(UInt16), + visually_complete Nullable(UInt16), + time_to_interactive Nullable(UInt16), + ttfb Nullable(UInt16) MATERIALIZED if (greaterOrEquals(response_start, request_start), minus(response_start, request_start), Null), + ttlb Nullable(UInt16) MATERIALIZED if (greaterOrEquals(response_end, request_start), minus(response_end, request_start), Null), + response_time Nullable(UInt16) MATERIALIZED if (greaterOrEquals(response_end, response_start), minus(response_end, response_start), Null), + dom_building_time Nullable(UInt16) MATERIALIZED if (greaterOrEquals(dom_content_loaded_event_start, response_end), minus(dom_content_loaded_event_start, response_end), Null), + dom_content_loaded_event_time Nullable(UInt16) MATERIALIZED if (greaterOrEquals(dom_content_loaded_event_end, dom_content_loaded_event_start), minus(dom_content_loaded_event_end, dom_content_loaded_event_start), Null), + load_event_time Nullable(UInt16) MATERIALIZED if (greaterOrEquals(load_event_end, load_event_start), minus(load_event_end, load_event_start), Null) ) ENGINE = MergeTree PARTITION BY toDate(datetime) ORDER BY (project_id, datetime) diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/performance.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/performance.sql index f05b79ff2..fa64967f4 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/performance.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/performance.sql @@ -1,30 +1,31 @@ -CREATE TABLE performance ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - min_fps UInt8, - avg_fps UInt8, - max_fps UInt8, - min_cpu UInt8, - avg_cpu UInt8, - max_cpu UInt8, - min_total_js_heap_size UInt64, - avg_total_js_heap_size UInt64, - max_total_js_heap_size UInt64, - min_used_js_heap_size UInt64, - avg_used_js_heap_size UInt64, - max_used_js_heap_size UInt64 +CREATE TABLE IF NOT EXISTS performance +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + min_fps UInt8, + avg_fps UInt8, + max_fps UInt8, + min_cpu UInt8, + avg_cpu UInt8, + max_cpu UInt8, + min_total_js_heap_size UInt64, + avg_total_js_heap_size UInt64, + max_total_js_heap_size UInt64, + min_used_js_heap_size UInt64, + avg_used_js_heap_size UInt64, + max_used_js_heap_size UInt64 ) ENGINE = MergeTree -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime) -TTL datetime + INTERVAL 1 MONTH; + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime) + TTL datetime + INTERVAL 1 MONTH; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/resources.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/resources.sql index 7b50443a3..cc2c7cd6d 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/resources.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/resources.sql @@ -1,31 +1,32 @@ -CREATE TABLE resources ( - session_id UInt64, - project_id UInt32, - tracker_version String, - rev_id Nullable(String), - user_uuid UUID, - user_os String, - user_os_version Nullable(String), - user_browser String, - user_browser_version Nullable(String), - user_device Nullable(String), - user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), - user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), - datetime DateTime, - url String, - url_host String MATERIALIZED lower(domain(url)), - url_hostpath String MATERIALIZED concat(url_host, lower(path(url))), - type Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4), - duration Nullable(UInt16), - ttfb Nullable(UInt16), - header_size Nullable(UInt16), - encoded_body_size Nullable(UInt32), - decoded_body_size Nullable(UInt32), - compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), - success UInt8, - method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), - status Nullable(UInt16) +CREATE TABLE IF NOT EXISTS resources +( + session_id UInt64, + project_id UInt32, + tracker_version String, + rev_id Nullable(String), + user_uuid UUID, + user_os String, + user_os_version Nullable(String), + user_browser String, + user_browser_version Nullable(String), + user_device Nullable(String), + user_device_type Enum8('other'=0, 'desktop'=1, 'mobile'=2), + user_country Enum8('UN'=-128, 'RW'=-127, 'SO'=-126, 'YE'=-125, 'IQ'=-124, 'SA'=-123, 'IR'=-122, 'CY'=-121, 'TZ'=-120, 'SY'=-119, 'AM'=-118, 'KE'=-117, 'CD'=-116, 'DJ'=-115, 'UG'=-114, 'CF'=-113, 'SC'=-112, 'JO'=-111, 'LB'=-110, 'KW'=-109, 'OM'=-108, 'QA'=-107, 'BH'=-106, 'AE'=-105, 'IL'=-104, 'TR'=-103, 'ET'=-102, 'ER'=-101, 'EG'=-100, 'SD'=-99, 'GR'=-98, 'BI'=-97, 'EE'=-96, 'LV'=-95, 'AZ'=-94, 'LT'=-93, 'SJ'=-92, 'GE'=-91, 'MD'=-90, 'BY'=-89, 'FI'=-88, 'AX'=-87, 'UA'=-86, 'MK'=-85, 'HU'=-84, 'BG'=-83, 'AL'=-82, 'PL'=-81, 'RO'=-80, 'XK'=-79, 'ZW'=-78, 'ZM'=-77, 'KM'=-76, 'MW'=-75, 'LS'=-74, 'BW'=-73, 'MU'=-72, 'SZ'=-71, 'RE'=-70, 'ZA'=-69, 'YT'=-68, 'MZ'=-67, 'MG'=-66, 'AF'=-65, 'PK'=-64, 'BD'=-63, 'TM'=-62, 'TJ'=-61, 'LK'=-60, 'BT'=-59, 'IN'=-58, 'MV'=-57, 'IO'=-56, 'NP'=-55, 'MM'=-54, 'UZ'=-53, 'KZ'=-52, 'KG'=-51, 'TF'=-50, 'HM'=-49, 'CC'=-48, 'PW'=-47, 'VN'=-46, 'TH'=-45, 'ID'=-44, 'LA'=-43, 'TW'=-42, 'PH'=-41, 'MY'=-40, 'CN'=-39, 'HK'=-38, 'BN'=-37, 'MO'=-36, 'KH'=-35, 'KR'=-34, 'JP'=-33, 'KP'=-32, 'SG'=-31, 'CK'=-30, 'TL'=-29, 'RU'=-28, 'MN'=-27, 'AU'=-26, 'CX'=-25, 'MH'=-24, 'FM'=-23, 'PG'=-22, 'SB'=-21, 'TV'=-20, 'NR'=-19, 'VU'=-18, 'NC'=-17, 'NF'=-16, 'NZ'=-15, 'FJ'=-14, 'LY'=-13, 'CM'=-12, 'SN'=-11, 'CG'=-10, 'PT'=-9, 'LR'=-8, 'CI'=-7, 'GH'=-6, 'GQ'=-5, 'NG'=-4, 'BF'=-3, 'TG'=-2, 'GW'=-1, 'MR'=0, 'BJ'=1, 'GA'=2, 'SL'=3, 'ST'=4, 'GI'=5, 'GM'=6, 'GN'=7, 'TD'=8, 'NE'=9, 'ML'=10, 'EH'=11, 'TN'=12, 'ES'=13, 'MA'=14, 'MT'=15, 'DZ'=16, 'FO'=17, 'DK'=18, 'IS'=19, 'GB'=20, 'CH'=21, 'SE'=22, 'NL'=23, 'AT'=24, 'BE'=25, 'DE'=26, 'LU'=27, 'IE'=28, 'MC'=29, 'FR'=30, 'AD'=31, 'LI'=32, 'JE'=33, 'IM'=34, 'GG'=35, 'SK'=36, 'CZ'=37, 'NO'=38, 'VA'=39, 'SM'=40, 'IT'=41, 'SI'=42, 'ME'=43, 'HR'=44, 'BA'=45, 'AO'=46, 'NA'=47, 'SH'=48, 'BV'=49, 'BB'=50, 'CV'=51, 'GY'=52, 'GF'=53, 'SR'=54, 'PM'=55, 'GL'=56, 'PY'=57, 'UY'=58, 'BR'=59, 'FK'=60, 'GS'=61, 'JM'=62, 'DO'=63, 'CU'=64, 'MQ'=65, 'BS'=66, 'BM'=67, 'AI'=68, 'TT'=69, 'KN'=70, 'DM'=71, 'AG'=72, 'LC'=73, 'TC'=74, 'AW'=75, 'VG'=76, 'VC'=77, 'MS'=78, 'MF'=79, 'BL'=80, 'GP'=81, 'GD'=82, 'KY'=83, 'BZ'=84, 'SV'=85, 'GT'=86, 'HN'=87, 'NI'=88, 'CR'=89, 'VE'=90, 'EC'=91, 'CO'=92, 'PA'=93, 'HT'=94, 'AR'=95, 'CL'=96, 'BO'=97, 'PE'=98, 'MX'=99, 'PF'=100, 'PN'=101, 'KI'=102, 'TK'=103, 'TO'=104, 'WF'=105, 'WS'=106, 'NU'=107, 'MP'=108, 'GU'=109, 'PR'=110, 'VI'=111, 'UM'=112, 'AS'=113, 'CA'=114, 'US'=115, 'PS'=116, 'RS'=117, 'AQ'=118, 'SX'=119, 'CW'=120, 'BQ'=121, 'SS'=122), + datetime DateTime, + url String, + url_host String MATERIALIZED lower(domain(url)), + url_hostpath String MATERIALIZED concat(url_host, lower(path(url))), + type Enum8('other'=-1, 'script'=0, 'stylesheet'=1, 'fetch'=2, 'img'=3, 'media'=4), + duration Nullable(UInt16), + ttfb Nullable(UInt16), + header_size Nullable(UInt16), + encoded_body_size Nullable(UInt32), + decoded_body_size Nullable(UInt32), + compression_ratio Nullable(Float32) MATERIALIZED divide(decoded_body_size, encoded_body_size), + success UInt8, + method Nullable(Enum8('GET' = 0, 'HEAD' = 1, 'POST' = 2, 'PUT' = 3, 'DELETE' = 4, 'CONNECT' = 5, 'OPTIONS' = 6, 'TRACE' = 7, 'PATCH' = 8)), + status Nullable(UInt16) ) ENGINE = MergeTree -PARTITION BY toDate(datetime) -ORDER BY (project_id, datetime) -TTL datetime + INTERVAL 1 MONTH; + PARTITION BY toDate(datetime) + ORDER BY (project_id, datetime) + TTL datetime + INTERVAL 1 MONTH; diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql index 54901bdc4..22cc6b876 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions.sql @@ -1,4 +1,4 @@ -CREATE TABLE sessions +CREATE TABLE IF NOT EXISTS sessions ( session_id UInt64, project_id UInt32, diff --git a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions_metadata.sql b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions_metadata.sql index a6d0382e6..ddf8aed01 100644 --- a/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions_metadata.sql +++ b/ee/scripts/helm/db/init_dbs/clickhouse/create/sessions_metadata.sql @@ -1,4 +1,4 @@ -CREATE TABLE sessions_metadata +CREATE TABLE IF NOT EXISTS sessions_metadata ( session_id UInt64, project_id UInt32, From eb065deb3ddb4e90d36bfdf923be2013119c7fa8 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 16:20:33 +0100 Subject: [PATCH 139/206] feat(api): custom metrics return full-metric on create and update --- api/chalicelib/core/custom_metrics.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index ffd911fea..62bddfbb3 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -56,8 +56,7 @@ def create(project_id, user_id, data: schemas.CreateCustomMetricsSchema): query ) r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return {"data": r} + return {"data": get(metric_id=r["metric_id"], project_id=project_id, user_id=user_id)} def __get_series_id(metric_id): @@ -135,8 +134,7 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche query ) r = cur.fetchone() - r = helper.dict_to_camel_case(r) - return r + return get(metric_id=metric_id, project_id=project_id, user_id=user_id) def get_all(project_id, user_id): From 200be22960013536482d2c6b73dbc47027c3b3d4 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Mon, 17 Jan 2022 18:16:46 +0100 Subject: [PATCH 140/206] Update README.md --- scripts/helmcharts/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/helmcharts/README.md b/scripts/helmcharts/README.md index b54ca9652..2e4475720 100644 --- a/scripts/helmcharts/README.md +++ b/scripts/helmcharts/README.md @@ -1,6 +1,3 @@ -- Initialize databases - - we've to pass the --wait flag, else the db installation won't be complete. and it'll break the db init. - ## Installation helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic From fef0af2a122ccbfd5722930cba106a072fe3bc8d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 18:22:01 +0100 Subject: [PATCH 141/206] feat(api): funnels fixes --- api/chalicelib/core/funnels.py | 40 ++++++++++++++++++---------------- api/routers/core.py | 19 ++++++++-------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index d73204c77..cb544a62f 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -1,6 +1,7 @@ import json import chalicelib.utils.helper +import schemas from chalicelib.core import events, significance, sessions from chalicelib.utils import dev from chalicelib.utils import helper, pg_client @@ -144,28 +145,28 @@ def delete(project_id, funnel_id, user_id): def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) -def get_sessions_on_the_fly(funnel_id, project_id, user_id, data): - data["events"] = filter_stages(data.get("events", [])) - if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) +def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): + data.events = filter_stages(data.events) + if len(data.events) == 0: + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), - start_date=data.get('startDate', None), - end_date=data.get('endDate', None)) - data = f["filter"] - return sessions.search2_pg(data=data, project_id=project_id, user_id=user_id) + get_start_end_time(filter_d=f["filter"], range_value=data.range_value, + start_date=data.startDate, end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) + return sessions.search2_pg(data=data, project_id=project_id, + user_id=user_id) -def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) +def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -175,10 +176,10 @@ def get_top_insights(project_id, funnel_id, range_value=None, start_date=None, e "totalDropDueToIssues": total_drop_due_to_issues}} -def get_top_insights_on_the_fly(funnel_id, project_id, data): +def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), @@ -192,8 +193,8 @@ def get_top_insights_on_the_fly(funnel_id, project_id, data): "totalDropDueToIssues": total_drop_due_to_issues}} -def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id) +def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -203,12 +204,12 @@ def get_issues(project_id, funnel_id, range_value=None, start_date=None, end_dat @dev.timed -def get_issues_on_the_fly(funnel_id, project_id, data): +def get_issues_on_the_fly(funnel_id, user_id, project_id, data): first_stage = data.get("firstStage") last_stage = data.get("lastStage") data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: - f = get(funnel_id=funnel_id, project_id=project_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=data.get("rangeValue", None), @@ -257,7 +258,8 @@ def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value= data = f["filter"] # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - issues = get_issues_on_the_fly(funnel_id=funnel_id, project_id=project_id, data=data).get("issues", {}) + issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data) \ + .get("issues", {}) issues = issues.get("significant", []) + issues.get("insignificant", []) issue = None for i in issues: diff --git a/api/routers/core.py b/api/routers/core.py index 792e4c739..857df1fa6 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -678,32 +678,31 @@ def get_possible_issue_types(projectId: int, context: schemas.CurrentContext = D @app.get('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) def get_funnel_insights(projectId: int, funnelId: int, rangeValue: str = None, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_top_insights(funnel_id=funnelId, project_id=projectId, - range_value=rangeValue, - start_date=startDate, - end_date=endDate) + return funnels.get_top_insights(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + range_value=rangeValue, start_date=startDate, end_date=endDate) @app.post('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) @app.put('/{projectId}/funnels/{funnelId}/insights', tags=["funnels"]) def get_funnel_insights_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelInsightsPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, project_id=projectId, data=data.dict()) + return funnels.get_top_insights_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + data=data.dict()) @app.get('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) def get_funnel_issues(projectId: int, funnelId, rangeValue: str = None, startDate: int = None, endDate: int = None, context: schemas.CurrentContext = Depends(OR_context)): - return funnels.get_issues(funnel_id=funnelId, project_id=projectId, - range_value=rangeValue, - start_date=startDate, end_date=endDate) + return funnels.get_issues(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + range_value=rangeValue, start_date=startDate, end_date=endDate) @app.post('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) @app.put('/{projectId}/funnels/{funnelId}/issues', tags=["funnels"]) def get_funnel_issues_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): - return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, project_id=projectId, data=data.dict())} + return {"data": funnels.get_issues_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, + data=data.dict())} @app.get('/{projectId}/funnels/{funnelId}/sessions', tags=["funnels"]) @@ -720,7 +719,7 @@ def get_funnel_sessions(projectId: int, funnelId: int, rangeValue: str = None, s def get_funnel_sessions_on_the_fly(projectId: int, funnelId: int, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): return {"data": funnels.get_sessions_on_the_fly(funnel_id=funnelId, user_id=context.user_id, project_id=projectId, - data=data.dict())} + data=data)} @app.get('/{projectId}/funnels/issues/{issueId}/sessions', tags=["funnels"]) From 11d94a179999ab05104e401ddaff0aec21b34a0a Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 17 Jan 2022 18:43:20 +0100 Subject: [PATCH 142/206] feat(api): funnels changes --- api/chalicelib/core/funnels.py | 29 +++++++++++++++-------------- api/routers/core.py | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index cb544a62f..281aeb422 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -12,23 +12,24 @@ REMOVE_KEYS = ["key", "_key", "startDate", "endDate"] ALLOW_UPDATE_FOR = ["name", "filter"] -def filter_stages(stages): - ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, - events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, - events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, - events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] - return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] +# def filter_stages(stages): +# ALLOW_TYPES = [events.event_type.CLICK.ui_type, events.event_type.INPUT.ui_type, +# events.event_type.LOCATION.ui_type, events.event_type.CUSTOM.ui_type, +# events.event_type.CLICK_IOS.ui_type, events.event_type.INPUT_IOS.ui_type, +# events.event_type.VIEW_IOS.ui_type, events.event_type.CUSTOM_IOS.ui_type, ] +# return [s for s in stages if s["type"] in ALLOW_TYPES and s.get("value") is not None] -def create(project_id, user_id, name, filter, is_public): +def create(project_id, user_id, name, filter: schemas.FunnelSearchPayloadSchema, is_public): helper.delete_keys_from_dict(filter, REMOVE_KEYS) - filter["events"] = filter_stages(stages=filter.get("events", [])) + # filter.events = filter_stages(stages=filter.events) with pg_client.PostgresClient() as cur: query = cur.mogrify("""\ INSERT INTO public.funnels (project_id, user_id, name, filter,is_public) VALUES (%(project_id)s, %(user_id)s, %(name)s, %(filter)s::jsonb,%(is_public)s) RETURNING *;""", - {"user_id": user_id, "project_id": project_id, "name": name, "filter": json.dumps(filter), + {"user_id": user_id, "project_id": project_id, "name": name, + "filter": json.dumps(filter.dict()), "is_public": is_public}) cur.execute( @@ -94,7 +95,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) if details: - row["filter"]["events"] = filter_stages(row["filter"]["events"]) + # row["filter"]["events"] = filter_stages(row["filter"]["events"]) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) @@ -153,7 +154,7 @@ def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=No def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): - data.events = filter_stages(data.events) + # data.events = filter_stages(data.events) if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -177,7 +178,7 @@ def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_dat def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): - data["events"] = filter_stages(data.get("events", [])) + # data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -207,7 +208,7 @@ def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None def get_issues_on_the_fly(funnel_id, user_id, project_id, data): first_stage = data.get("firstStage") last_stage = data.get("lastStage") - data["events"] = filter_stages(data.get("events", [])) + # data["events"] = filter_stages(data.get("events", [])) if len(data["events"]) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: @@ -243,7 +244,7 @@ def get(funnel_id, project_id, user_id): return None f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) - f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) return f diff --git a/api/routers/core.py b/api/routers/core.py index 857df1fa6..abaa60dea 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -645,7 +645,7 @@ def add_funnel(projectId: int, data: schemas.FunnelSchema = Body(...), return funnels.create(project_id=projectId, user_id=context.user_id, name=data.name, - filter=data.filter.dict(), + filter=data.filter, is_public=data.is_public) From b3a78aeca887a43c6673153b79021ed433e285c2 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 18 Jan 2022 12:39:41 +0100 Subject: [PATCH 143/206] feat(api): fixed update funnels --- api/chalicelib/core/funnels.py | 5 +++-- api/chalicelib/core/significance.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 281aeb422..22ab73cba 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -2,7 +2,7 @@ import json import chalicelib.utils.helper import schemas -from chalicelib.core import events, significance, sessions +from chalicelib.core import significance, sessions from chalicelib.utils import dev from chalicelib.utils import helper, pg_client from chalicelib.utils.TimeUTC import TimeUTC @@ -61,7 +61,8 @@ def update(funnel_id, user_id, project_id, name=None, filter=None, is_public=Non AND project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) RETURNING *;""", {"user_id": user_id, "funnel_id": funnel_id, "name": name, - "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public}) + "filter": json.dumps(filter) if filter is not None else None, "is_public": is_public, + "project_id": project_id}) # print("--------------------") # print(query) # print("--------------------") diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index 9fc55065f..d81378ddb 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -223,9 +223,9 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: params = {"project_id": project_id, "startTimestamp": filter_d["startDate"], "endTimestamp": filter_d["endDate"], "issueTypes": tuple(filter_issues), **values} with pg_client.PostgresClient() as cur: - print("---------------------------------------------------") - print(cur.mogrify(n_stages_query, params)) - print("---------------------------------------------------") + # print("---------------------------------------------------") + # print(cur.mogrify(n_stages_query, params)) + # print("---------------------------------------------------") cur.execute(cur.mogrify(n_stages_query, params)) rows = cur.fetchall() return rows From be2b1708b719936db648d52a56bb328b42b9fe27 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 18 Jan 2022 14:08:58 +0100 Subject: [PATCH 144/206] feat(api): fixed get funnel-sessions-issues --- api/chalicelib/core/funnels.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 22ab73cba..835a655f4 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -250,17 +250,19 @@ def get(funnel_id, project_id, user_id): @dev.timed -def search_by_issue(user_id, project_id, funnel_id, issue_id, data, range_value=None, start_date=None, end_date=None): - if len(data.get("events", [])) == 0: +def search_by_issue(user_id, project_id, funnel_id, issue_id, data: schemas.FunnelSearchPayloadSchema, range_value=None, + start_date=None, end_date=None): + if len(data.events) == 0: f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) if f is None: return {"errors": ["funnel not found"]} - get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.get('startDate', start_date), - end_date=data.get('endDate', end_date)) - data = f["filter"] + data.startDate = data.startDate if data.startDate is not None else start_date + data.endDate = data.endDate if data.endDate is not None else end_date + get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=data.startDate, + end_date=data.endDate) + data = schemas.FunnelSearchPayloadSchema.parse_obj(f["filter"]) - # insights, total_drop_due_to_issues = significance.get_top_insights(filter_d=data, project_id=project_id) - issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data) \ + issues = get_issues_on_the_fly(funnel_id=funnel_id, user_id=user_id, project_id=project_id, data=data.dict()) \ .get("issues", {}) issues = issues.get("significant", []) + issues.get("insignificant", []) issue = None From e742955fd01e75ef2d73a78b6bd211fb490fc05d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 18 Jan 2022 14:19:16 +0100 Subject: [PATCH 145/206] feat(api): funnel changes feat(api): reduce DB pool initial size --- api/.env.default | 2 +- api/routers/core.py | 2 +- ee/api/.env.default | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/.env.default b/api/.env.default index 3b05ce3d2..8f4ddc01c 100644 --- a/api/.env.default +++ b/api/.env.default @@ -35,7 +35,7 @@ pg_password=asayerPostgres pg_port=5432 pg_user=postgres pg_timeout=30 -pg_minconn=50 +pg_minconn=45 put_S3_TTL=20 sentryURL= sessions_bucket=mobs diff --git a/api/routers/core.py b/api/routers/core.py index abaa60dea..9c5bec86e 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -739,7 +739,7 @@ def get_funnel_issue_sessions(projectId: int, funnelId: int, issueId: str, data: schemas.FunnelSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): data = funnels.search_by_issue(project_id=projectId, user_id=context.user_id, issue_id=issueId, - funnel_id=funnelId, data=data.dict()) + funnel_id=funnelId, data=data) if "errors" in data: return data if data.get("issue") is None: diff --git a/ee/api/.env.default b/ee/api/.env.default index 6fff1793c..cec7e59a4 100644 --- a/ee/api/.env.default +++ b/ee/api/.env.default @@ -38,11 +38,13 @@ jwt_exp_delta_seconds=2592000 jwt_issuer=openreplay-default-ee jwt_secret="SET A RANDOM STRING HERE" peers=http://utilities-openreplay.app.svc.cluster.local:9000/assist/%s/peers -pg_dbname=app -pg_host=127.0.0.1 -pg_password= -pg_port=9202 -pg_user= +pg_dbname=postgres +pg_host=postgresql.db.svc.cluster.local +pg_password=asayerPostgres +pg_port=5432 +pg_user=postgres +pg_timeout=30 +pg_minconn=45 put_S3_TTL=20 sentryURL= sessions_bucket=mobs From 8037879e4347dcc4650dba1ddeae031ac355512f Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 00:10:31 +0530 Subject: [PATCH 146/206] fix(install): kafka ssl topic creation Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/files/kafka.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/files/kafka.sh b/scripts/helmcharts/openreplay/files/kafka.sh index 9105f3b00..1c811eb5d 100644 --- a/scripts/helmcharts/openreplay/files/kafka.sh +++ b/scripts/helmcharts/openreplay/files/kafka.sh @@ -13,7 +13,7 @@ topics=( touch /tmp/config.txt if [[ $KAFKA_SSL == "true" ]]; then - echo 'securty.protocol=SSL' > /tmp/config.txt + echo 'security.protocol=SSL' > /tmp/config.txt fi function init() { From 2a21d2eeb60201d74e4d08acd0e312ca0486fef5 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 00:10:31 +0530 Subject: [PATCH 147/206] fix(install): kafka ssl topic creation Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/files/kafka.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/files/kafka.sh b/scripts/helmcharts/openreplay/files/kafka.sh index 9105f3b00..1c811eb5d 100644 --- a/scripts/helmcharts/openreplay/files/kafka.sh +++ b/scripts/helmcharts/openreplay/files/kafka.sh @@ -13,7 +13,7 @@ topics=( touch /tmp/config.txt if [[ $KAFKA_SSL == "true" ]]; then - echo 'securty.protocol=SSL' > /tmp/config.txt + echo 'security.protocol=SSL' > /tmp/config.txt fi function init() { From 14b4d87943fd12fb3835d006bd18f8788976cc15 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 22:47:06 +0530 Subject: [PATCH 148/206] chore(helm): using minio keys to initialize minio rather than s3. Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/templates/job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index 74e2cadcc..4e0cdebf4 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -81,9 +81,9 @@ spec: - name: CHART_APP_VERSION value: "{{ .Chart.AppVersion }}" - name: MINIO_ACCESS_KEY - value: "{{ .Values.global.s3.accessKey }}" + value: "{{ .Values.minio.global.minio.accessKey }}" - name: MINIO_SECRET_KEY - value: "{{ .Values.global.s3.secretKey }}" + value: "{{ .Values.minio.global.minio.secretKey }}" command: - /bin/bash - /opt/migrations/dbops.sh From 1834806f4a8e8f24911e8677fe4c02345b121b73 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 22:47:06 +0530 Subject: [PATCH 149/206] chore(helm): using minio keys to initialize minio rather than s3. Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/openreplay/templates/job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/openreplay/templates/job.yaml b/scripts/helmcharts/openreplay/templates/job.yaml index 74e2cadcc..4e0cdebf4 100644 --- a/scripts/helmcharts/openreplay/templates/job.yaml +++ b/scripts/helmcharts/openreplay/templates/job.yaml @@ -81,9 +81,9 @@ spec: - name: CHART_APP_VERSION value: "{{ .Chart.AppVersion }}" - name: MINIO_ACCESS_KEY - value: "{{ .Values.global.s3.accessKey }}" + value: "{{ .Values.minio.global.minio.accessKey }}" - name: MINIO_SECRET_KEY - value: "{{ .Values.global.s3.secretKey }}" + value: "{{ .Values.minio.global.minio.secretKey }}" command: - /bin/bash - /opt/migrations/dbops.sh From 1fb6e470aab6bd8816595636f96166d8696cfd72 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 22:51:17 +0530 Subject: [PATCH 150/206] docs(helm): update vars.yaml examples Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 50bf9edd2..e8ef20bf7 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -87,14 +87,14 @@ chalice: # limits: # cpu: 1 -# nginx-ingress: +## Changes to nginx # -### If you want to redirect nginx http to https +# nginx-ingress: # customServerConfigs: | +# # Redirecting http to https # return 301 https://$host$request_uri; # ### Change the ssl certificates -# ### Public certificate ( content from site.crt, mind the indentation ) # ssl_certificate: |- # -----BEGIN CERTIFICATE----- @@ -109,10 +109,5 @@ chalice: # TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T # e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP # 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 -# WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa -# 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb -# esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn -# PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 -# PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 # 7qINKoGovqXB1oAdopIl1z64e7MWVE4= # -----END PRIVATE KEY----- From 833e6000ae7bb6cdd16dc3baee294d1ba66562b6 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 19 Jan 2022 22:51:17 +0530 Subject: [PATCH 151/206] docs(helm): update vars.yaml examples Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 50bf9edd2..e8ef20bf7 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -87,14 +87,14 @@ chalice: # limits: # cpu: 1 -# nginx-ingress: +## Changes to nginx # -### If you want to redirect nginx http to https +# nginx-ingress: # customServerConfigs: | +# # Redirecting http to https # return 301 https://$host$request_uri; # ### Change the ssl certificates -# ### Public certificate ( content from site.crt, mind the indentation ) # ssl_certificate: |- # -----BEGIN CERTIFICATE----- @@ -109,10 +109,5 @@ chalice: # TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T # e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP # 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 -# WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa -# 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb -# esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn -# PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 -# PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 # 7qINKoGovqXB1oAdopIl1z64e7MWVE4= # -----END PRIVATE KEY----- From 2a4badf94d8259c4b2066c267e1db7600b13fded Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Thu, 20 Jan 2022 20:47:13 +0100 Subject: [PATCH 152/206] Added captcha env variables --- scripts/helmcharts/vars.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index e8ef20bf7..e4928a786 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -67,6 +67,8 @@ global: chalice: env: jwt_secret: "SetARandomStringHere" + # CAPTCHA_SERVER: '' + # CAPTCHA_KEY: '' # SAML2_MD_URL: '' # idp_entityId: '' # idp_sso_url: '' From e01c3ccf04f8a161784872b8e9a5d13dbf623b81 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 11:00:24 +0100 Subject: [PATCH 153/206] Update LICENSE.md --- ee/LICENSE.md | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/ee/LICENSE.md b/ee/LICENSE.md index d99b63d76..5f6043f8f 100644 --- a/ee/LICENSE.md +++ b/ee/LICENSE.md @@ -1,36 +1,4 @@ The OpenReplay Enterprise license (the “Enterprise License”) -Copyright (c) 2021 Asayer SAS. +Copyright (c) 2022 Asayer SAS. -With regard to the OpenReplay Software: - -This software and associated documentation files (the "Software") may only be -used in production, if you (and any entity that you represent) have agreed to, -and are in compliance with, the OpenReplay Subscription Terms of Service, available -at https://openreplay.com/terms.html (the “Enterprise Edition”), or other -agreement governing the use of the Software, as agreed by you and OpenReplay, -and otherwise have a valid OpenReplay Enterprise license for the -correct usage. Subject to the foregoing sentence, you are free to -modify this Software and publish patches to the Software. You agree that OpenReplay -and/or its licensors (as applicable) retain all right, title and interest in and -to all such modifications and/or patches, and all such modifications and/or -patches may only be used, copied, modified, displayed, distributed, or otherwise -exploited with a valid OpenReplay Enterprise license for the correct -number of user seats and profiles. Notwithstanding the foregoing, you may copy and modify -the Software for development and testing purposes, without requiring a -subscription. You agree that OpenReplay and/or its licensors (as applicable) retain -all right, title and interest in and to all such modifications. You are not -granted any other rights beyond what is expressly stated herein. Subject to the -foregoing, it is forbidden to copy, merge, publish, distribute, sublicense, -and/or sell the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For all third party components incorporated into the OpenReplay Software, those -components are licensed under the original license provided by the owner of the -applicable component. +To license the Enterprise Edition of OpenReplay, and take advantage of its additional features, functionality and support, you must agree to the terms of the OpenReplay Enterprise License Agreement. Please contact OpenReplay at [sales@openreplay.com](mailto:sales@openreplay.com). From b422eb52107ca60b0e5435b19e333f2f1c06e07e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 12:04:54 +0100 Subject: [PATCH 154/206] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 940c97860..406fe9608 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Asayer SAS. +Copyright (c) 2022 Asayer SAS. Portions of this software are licensed as follows: From 4951b3105bb0847ab035880d37bafdb997eb8e73 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 14:25:11 +0100 Subject: [PATCH 155/206] Update third-party.md --- third-party.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/third-party.md b/third-party.md index a26f646c3..98aefe5b0 100644 --- a/third-party.md +++ b/third-party.md @@ -1,4 +1,4 @@ -## Licenses (as of January 16, 2022) +## Licenses (as of January 21, 2022) Below is the list of dependencies used in OpenReplay software. Licenses may change between versions, so please keep this up to date with every new library you use. @@ -84,10 +84,9 @@ Below is the list of dependencies used in OpenReplay software. Licenses may chan | redux-immutable | BSD3 | JavaScript | | redux-thunk | MIT | JavaScript | | semantic-ui-react | MIT | JavaScript | -| socket.io-client | MIT | JavaScript | +| socketio | MIT | JavaScript | | source-map | BSD3 | JavaScript | | aws-sdk | Apache2 | JavaScript | | serverless | MIT | JavaScript | | lib/pq | MIT | Go | | peerjs | MIT | JavaScript | -| antonmedv/finder | MIT | JavaScript | From 293619e61e38eb50f6ad2c0890f7445c7a65fb1e Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 14:25:18 +0100 Subject: [PATCH 156/206] feat(api): EE-SSO extra endpoint for response processing using tenantKey --- ee/api/routers/saml.py | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 50723a1db..73b0e7393 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -43,6 +43,8 @@ async def process_sso_assertion(request: Request): user_data = auth.get_attributes() elif auth.get_settings().is_debug_active(): error_reason = auth.get_last_error_reason() + print("SAML2 error:") + print(error_reason) return {"errors": [error_reason]} email = auth.get_nameid() @@ -108,6 +110,88 @@ async def process_sso_assertion(request: Request): headers={'Location': SAML2_helper.get_landing_URL(jwt)}) +@public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) +async def process_sso_assertion_tk(tenantKey: str, request: Request): + req = await prepare_request(request=request) + session = req["cookie"]["session"] + auth = init_saml_auth(req) + + request_id = None + if 'AuthNRequestID' in session: + request_id = session['AuthNRequestID'] + + auth.process_response(request_id=request_id) + errors = auth.get_errors() + user_data = {} + if len(errors) == 0: + if 'AuthNRequestID' in session: + del session['AuthNRequestID'] + user_data = auth.get_attributes() + elif auth.get_settings().is_debug_active(): + error_reason = auth.get_last_error_reason() + print("SAML2 error:") + print(error_reason) + return {"errors": [error_reason]} + + email = auth.get_nameid() + print("received nameId:") + print(email) + existing = users.get_by_email_only(auth.get_nameid()) + + internal_id = next(iter(user_data.get("internalId", [])), None) + + t = tenants.get_by_tenant_key(tenantKey) + if t is None: + print("invalid tenantKey, please copy the correct value from Preferences > Account") + return {"errors": ["invalid tenantKey, please copy the correct value from Preferences > Account"]} + print(user_data) + role_name = user_data.get("role", []) + if len(role_name) == 0: + print("No role specified, setting role to member") + role_name = ["member"] + role_name = role_name[0] + role = roles.get_role_by_name(tenant_id=t['tenantId'], name=role_name) + if role is None: + return {"errors": [f"role {role_name} not found, please create it in openreplay first"]} + + admin_privileges = user_data.get("adminPrivileges", []) + admin_privileges = not (len(admin_privileges) == 0 + or admin_privileges[0] is None + or admin_privileges[0].lower() == "false") + + if existing is None: + deleted = users.get_deleted_user_by_email(auth.get_nameid()) + if deleted is not None: + print("== restore deleted user ==") + users.restore_sso_user(user_id=deleted["userId"], tenant_id=t['tenantId'], email=email, + admin=admin_privileges, origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + print("== new user ==") + users.create_sso_user(tenant_id=t['tenantId'], email=email, admin=admin_privileges, + origin=SAML2_helper.get_saml2_provider(), + name=" ".join(user_data.get("firstName", []) + user_data.get("lastName", [])), + internal_id=internal_id, role_id=role["roleId"]) + else: + if t['tenantId'] != existing["tenantId"]: + print("user exists for a different tenant") + return {"errors": ["user exists for a different tenant"]} + if existing.get("origin") is None: + print(f"== migrating user to {SAML2_helper.get_saml2_provider()} ==") + users.update(tenant_id=t['tenantId'], user_id=existing["id"], + changes={"origin": SAML2_helper.get_saml2_provider(), "internal_id": internal_id}) + expiration = auth.get_session_expiration() + expiration = expiration if expiration is not None and expiration > 10 * 60 \ + else int(config("sso_exp_delta_seconds", cast=int, default=24 * 60 * 60)) + jwt = users.authenticate_sso(email=email, internal_id=internal_id, exp=expiration) + if jwt is None: + return {"errors": ["null JWT"]} + return Response( + status_code=status.HTTP_302_FOUND, + headers={'Location': SAML2_helper.get_landing_URL(jwt)}) + + @public_app.get('/sso/saml2/sls', tags=["saml2"]) async def process_sls_assertion(request: Request): req = await prepare_request(request=request) @@ -143,6 +227,7 @@ async def process_sls_assertion(request: Request): return RedirectResponse(url=config("SITE_URL")) +@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) @public_app.get('/sso/saml2/metadata', tags=["saml2"]) async def saml2_metadata(request: Request): req = await prepare_request(request=request) From 2482efb66d5ddc76e369cec9bdde7551316ef84b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 14:59:39 +0100 Subject: [PATCH 157/206] feat(api): EE-SSO remove /tenantKey from ACS endpoint --- ee/api/chalicelib/utils/SAML2_helper.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index a2a4e1e6e..6a1ee4eca 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -86,12 +86,22 @@ async def prepare_request(request: Request): session = {} # If server is behind proxys or balancers use the HTTP_X_FORWARDED fields headers = request.headers - url_data = urlparse('%s://%s' % (headers.get('x-forwarded-proto', 'http'), headers['host'])) + proto = headers.get('x-forwarded-proto', 'http') + url_data = urlparse('%s://%s' % (proto, headers['host'])) + path = request.url.path + # remove / from the /acs/ + if path.endswith("/"): + path = path[:-1] + # remove /{tenantKey} from /acs/{tenantKey} + if not path.endswith("/acs"): + parts = path.split("/") + if len(parts) > 2 and parts[-2] == "acs": + path = "/".join(parts[:-1]) return { - 'https': 'on' if request.headers.get('x-forwarded-proto', 'http') == 'https' else 'off', + 'https': 'on' if proto == 'https' else 'off', 'http_host': request.headers['host'], 'server_port': url_data.port, - 'script_name': "/api" + request.url.path, + 'script_name': "/api" + path, 'get_data': request.args.copy(), # Uncomment if using ADFS as IdP, https://github.com/onelogin/python-saml/pull/144 # 'lowercase_urlencoding': True, From 1ab8c1068784059658b8c39096695ddf2888902d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 15:26:44 +0100 Subject: [PATCH 158/206] feat(api): EE-SSO log request --- ee/api/routers/saml.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 73b0e7393..117960b50 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -113,6 +113,9 @@ async def process_sso_assertion(request: Request): @public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) async def process_sso_assertion_tk(tenantKey: str, request: Request): req = await prepare_request(request=request) + print("------------") + print(req) + print("------------") session = req["cookie"]["session"] auth = init_saml_auth(req) From 320efff88ecfb14e154e04b4f0185bf5ed94575b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 15:46:46 +0100 Subject: [PATCH 159/206] feat(api): EE allow live editing --- api/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/entrypoint.sh b/api/entrypoint.sh index 60fefb5c0..a092737be 100755 --- a/api/entrypoint.sh +++ b/api/entrypoint.sh @@ -1,2 +1,2 @@ #!/bin/bash -uvicorn app:app --host 0.0.0.0 +uvicorn app:app --host 0.0.0.0 --reload From 8639f893792a37bae8819a0fc721b3f7a17d9c67 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 15:50:15 +0100 Subject: [PATCH 160/206] feat(api): EE allow live editing --- ee/api/.gitignore | 2 +- ee/api/entrypoint.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 ee/api/entrypoint.sh diff --git a/ee/api/.gitignore b/ee/api/.gitignore index 8afea0ab6..f1ff9550b 100644 --- a/ee/api/.gitignore +++ b/ee/api/.gitignore @@ -253,7 +253,7 @@ Pipfile /db_changes.sql /Dockerfile.bundle /entrypoint.bundle.sh -/entrypoint.sh +#/entrypoint.sh /chalicelib/core/heatmaps.py /routers/subs/insights.py /schemas.py diff --git a/ee/api/entrypoint.sh b/ee/api/entrypoint.sh new file mode 100755 index 000000000..a092737be --- /dev/null +++ b/ee/api/entrypoint.sh @@ -0,0 +1,2 @@ +#!/bin/bash +uvicorn app:app --host 0.0.0.0 --reload From 1f8b9e2d8845099b353522deddcb819fbe37ae23 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 22:03:49 +0530 Subject: [PATCH 161/206] chore(nginx): Adding protocol scheme forwarding Signed-off-by: Rajesh Rajendran --- .../charts/nginx-ingress/templates/configMap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml index f635b270c..77100c35c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml @@ -62,7 +62,7 @@ data: proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $origin_proto; proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; } location /assist/ { @@ -134,6 +134,10 @@ data: default upgrade; '' close; } + map $http_x_forwarded_proto $origin_proto { + default $http_x_forwarded_proto; + '' $scheme; + } server { listen 80 default_server; listen [::]:80 default_server; From c181d2198a4abd847210a05fbff2c436d09b3683 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 22:03:49 +0530 Subject: [PATCH 162/206] chore(nginx): Adding protocol scheme forwarding Signed-off-by: Rajesh Rajendran --- .../charts/nginx-ingress/templates/configMap.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml index f635b270c..77100c35c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml @@ -62,7 +62,7 @@ data: proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Proto $origin_proto; proxy_pass http://chalice-openreplay.app.svc.cluster.local:8000; } location /assist/ { @@ -134,6 +134,10 @@ data: default upgrade; '' close; } + map $http_x_forwarded_proto $origin_proto { + default $http_x_forwarded_proto; + '' $scheme; + } server { listen 80 default_server; listen [::]:80 default_server; From b0e2fd127aa20ca231176aad4d9dbed61d451640 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:02:36 +0100 Subject: [PATCH 163/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index e4928a786..91de773e0 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -67,8 +67,8 @@ global: chalice: env: jwt_secret: "SetARandomStringHere" - # CAPTCHA_SERVER: '' - # CAPTCHA_KEY: '' + # captcha_server: '' + # captcha_key: '' # SAML2_MD_URL: '' # idp_entityId: '' # idp_sso_url: '' From f519700feabf8d115832c91e0030dc1b3d7f1e9d Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 18:43:12 +0100 Subject: [PATCH 164/206] feat(api): EE-SSO include tenantKey in ACS URL feat(helm): idp_sp_tk for chalice env vars --- ee/api/chalicelib/utils/SAML2_helper.py | 29 ++++++++++--------- ee/api/routers/saml.py | 9 +++--- scripts/helm/app/chalice.yaml | 1 + .../openreplay/charts/chalice/values.yaml | 1 + scripts/helmcharts/vars.yaml | 1 + 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index 6a1ee4eca..86a6c4683 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -12,11 +12,11 @@ SAML2 = { "sp": { "entityId": config("SITE_URL") + "/api/sso/saml2/metadata/", "assertionConsumerService": { - "url": config("SITE_URL") + "/api/sso/saml2/acs", + "url": config("SITE_URL") + "/api/sso/saml2/acs/", "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" }, "singleLogoutService": { - "url": config("SITE_URL") + "/api/sso/saml2/sls", + "url": config("SITE_URL") + "/api/sso/saml2/sls/", "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" }, "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress", @@ -25,6 +25,12 @@ SAML2 = { }, "idp": None } + +# in case tenantKey is included in the URL +sp_acs = config("idp_sp_tk", default="") +if sp_acs is not None and len(sp_acs) > 0: + SAML2["sp"]["assertionConsumerService"]["url"] += sp_acs + "/" + idp = None # SAML2 config handler if config("SAML2_MD_URL", default=None) is not None and len(config("SAML2_MD_URL")) > 0: @@ -60,12 +66,9 @@ else: def init_saml_auth(req): # auth = OneLogin_Saml2_Auth(req, custom_base_path=environ['SAML_PATH']) - if idp is None: raise Exception("No SAML2 config provided") - auth = OneLogin_Saml2_Auth(req, old_settings=SAML2) - - return auth + return OneLogin_Saml2_Auth(req, old_settings=SAML2) async def prepare_request(request: Request): @@ -87,16 +90,14 @@ async def prepare_request(request: Request): # If server is behind proxys or balancers use the HTTP_X_FORWARDED fields headers = request.headers proto = headers.get('x-forwarded-proto', 'http') + if headers.get('x-forwarded-proto') is not None: + print(f"x-forwarded-proto: {proto}") url_data = urlparse('%s://%s' % (proto, headers['host'])) path = request.url.path - # remove / from the /acs/ - if path.endswith("/"): - path = path[:-1] - # remove /{tenantKey} from /acs/{tenantKey} - if not path.endswith("/acs"): - parts = path.split("/") - if len(parts) > 2 and parts[-2] == "acs": - path = "/".join(parts[:-1]) + # add / to /acs + if not path.endswith("/"): + path = path + '/' + return { 'https': 'on' if proto == 'https' else 'off', 'http_host': request.headers['host'], diff --git a/ee/api/routers/saml.py b/ee/api/routers/saml.py index 117960b50..ee0f0333b 100644 --- a/ee/api/routers/saml.py +++ b/ee/api/routers/saml.py @@ -16,6 +16,7 @@ from starlette import status @public_app.get("/sso/saml2", tags=["saml2"]) +@public_app.get("/sso/saml2/", tags=["saml2"]) async def start_sso(request: Request): request.path = '' req = await prepare_request(request=request) @@ -25,6 +26,7 @@ async def start_sso(request: Request): @public_app.post('/sso/saml2/acs', tags=["saml2"]) +@public_app.post('/sso/saml2/acs/', tags=["saml2"]) async def process_sso_assertion(request: Request): req = await prepare_request(request=request) session = req["cookie"]["session"] @@ -111,11 +113,9 @@ async def process_sso_assertion(request: Request): @public_app.post('/sso/saml2/acs/{tenantKey}', tags=["saml2"]) +@public_app.post('/sso/saml2/acs/{tenantKey}/', tags=["saml2"]) async def process_sso_assertion_tk(tenantKey: str, request: Request): req = await prepare_request(request=request) - print("------------") - print(req) - print("------------") session = req["cookie"]["session"] auth = init_saml_auth(req) @@ -196,6 +196,7 @@ async def process_sso_assertion_tk(tenantKey: str, request: Request): @public_app.get('/sso/saml2/sls', tags=["saml2"]) +@public_app.get('/sso/saml2/sls/', tags=["saml2"]) async def process_sls_assertion(request: Request): req = await prepare_request(request=request) session = req["cookie"]["session"] @@ -230,8 +231,8 @@ async def process_sls_assertion(request: Request): return RedirectResponse(url=config("SITE_URL")) -@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) @public_app.get('/sso/saml2/metadata', tags=["saml2"]) +@public_app.get('/sso/saml2/metadata/', tags=["saml2"]) async def saml2_metadata(request: Request): req = await prepare_request(request=request) auth = init_saml_auth(req) diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index fcbea8ed6..09d015401 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -64,5 +64,6 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' + idp_sp_tk: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index e5c579739..8e0080746 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -94,6 +94,7 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' + idp_sp_tk: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index e8ef20bf7..12b4971f8 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -73,6 +73,7 @@ chalice: # idp_x509cert: '' # idp_sls_url: '' # idp_name: '' + # idp_sp_tk: '' # If you want to override something From f6a318e50f2bd33e8e5ccbb0c2bde0f350f0972e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:46:02 +0100 Subject: [PATCH 165/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 91de773e0..0ca05ba87 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -84,10 +84,14 @@ chalice: # For example: # -# alerts: +# http: # resources: # limits: -# cpu: 1 +# cpu: 1024m +# memory: 4096Mi +# requests: +# cpu: 512m +# memory: 2056Mi ## Changes to nginx # From c9a016988d1b7364d85d80fa6f97916cd9514b9b Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:46:17 +0100 Subject: [PATCH 166/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 0ca05ba87..53272762b 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -100,20 +100,3 @@ chalice: # # Redirecting http to https # return 301 https://$host$request_uri; # -### Change the ssl certificates -### Public certificate ( content from site.crt, mind the indentation ) -# ssl_certificate: |- -# -----BEGIN CERTIFICATE----- -# MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL -# BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 -# NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh -# -----END CERTIFICATE----- -# -### Private certificate ( content from site.key, mind the indentation. ) -# ssl_privatekey: |- -# -----BEGIN PRIVATE KEY----- -# TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T -# e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP -# 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 -# 7qINKoGovqXB1oAdopIl1z64e7MWVE4= -# -----END PRIVATE KEY----- From 414a6da84d22eec9addca66351f43536449eabf0 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Fri, 21 Jan 2022 23:04:43 +0100 Subject: [PATCH 167/206] feat(api): EE-SSO include idp_tenantKey in ACS URL/path feat(helm): idp_tenantKey for chalice env vars --- ee/api/chalicelib/utils/SAML2_helper.py | 2 +- scripts/helm/app/chalice.yaml | 2 +- scripts/helmcharts/openreplay/charts/chalice/values.yaml | 2 +- scripts/helmcharts/vars.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ee/api/chalicelib/utils/SAML2_helper.py b/ee/api/chalicelib/utils/SAML2_helper.py index 86a6c4683..c00081d2c 100644 --- a/ee/api/chalicelib/utils/SAML2_helper.py +++ b/ee/api/chalicelib/utils/SAML2_helper.py @@ -27,7 +27,7 @@ SAML2 = { } # in case tenantKey is included in the URL -sp_acs = config("idp_sp_tk", default="") +sp_acs = config("idp_tenantKey", default="") if sp_acs is not None and len(sp_acs) > 0: SAML2["sp"]["assertionConsumerService"]["url"] += sp_acs + "/" diff --git a/scripts/helm/app/chalice.yaml b/scripts/helm/app/chalice.yaml index 09d015401..2d6b53ead 100644 --- a/scripts/helm/app/chalice.yaml +++ b/scripts/helm/app/chalice.yaml @@ -64,6 +64,6 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' - idp_sp_tk: '' + idp_tenantKey: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/openreplay/charts/chalice/values.yaml b/scripts/helmcharts/openreplay/charts/chalice/values.yaml index 8e0080746..5e76420e8 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/values.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/values.yaml @@ -94,7 +94,7 @@ env: idp_x509cert: '' idp_sls_url: '' idp_name: '' - idp_sp_tk: '' + idp_tenantKey: '' assist_secret: '' iceServers: '' diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 12b4971f8..46d52911c 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -73,7 +73,7 @@ chalice: # idp_x509cert: '' # idp_sls_url: '' # idp_name: '' - # idp_sp_tk: '' + # idp_tenantKey: '' # If you want to override something From 9025433cf5143b50570e24457efb7e32c0da5796 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 11:24:23 +0100 Subject: [PATCH 168/206] feat(api): alerts/triggers endpoint --- api/chalicelib/core/alerts.py | 13 ++++++++++++- api/chalicelib/core/custom_metrics.py | 13 +++++++------ api/routers/core.py | 11 ++++++----- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/api/chalicelib/core/alerts.py b/api/chalicelib/core/alerts.py index c701f0ce0..6fe799c19 100644 --- a/api/chalicelib/core/alerts.py +++ b/api/chalicelib/core/alerts.py @@ -1,11 +1,12 @@ import json +import logging import time import schemas from chalicelib.core import notifications, slack, webhook from chalicelib.utils import pg_client, helper, email_helper from chalicelib.utils.TimeUTC import TimeUTC -import logging + def get(id): with pg_client.PostgresClient() as cur: @@ -157,3 +158,13 @@ def delete(project_id, alert_id): {"alert_id": alert_id, "project_id": project_id}) ) return {"data": {"state": "success"}} + + +def get_predefined_values(): + values = [e.value for e in schemas.AlertColumn] + values = [{"name": v, "value": v, + "unit": "count" if v.endswith(".count") else "ms", + "predefined": True, + "metricId": None, + "seriesId": None} for v in values] + return values diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 62bddfbb3..787b19837 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -205,17 +205,18 @@ def get_series_for_alert(project_id, user_id): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( - """SELECT metric_id, - series_id, - metrics.name AS metric_name, - metric_series.name AS series_name, - index AS series_index + """SELECT series_id AS value, + metrics.name || '.' || (COALESCE(metric_series.name, 'series ' || index)) || '.count' AS name, + 'count' AS unit, + FALSE AS predefined, + metric_id, + series_id FROM metric_series INNER JOIN metrics USING (metric_id) WHERE metrics.deleted_at ISNULL AND metrics.project_id = %(project_id)s AND (user_id = %(user_id)s OR is_public) - ORDER BY metric_name, series_index, series_name;""", + ORDER BY name;""", {"project_id": project_id, "user_id": user_id} ) ) diff --git a/api/routers/core.py b/api/routers/core.py index 9c5bec86e..129ae9cc9 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -621,6 +621,12 @@ def get_all_alerts(projectId: int, context: schemas.CurrentContext = Depends(OR_ return {"data": alerts.get_all(projectId)} +@app.get('/{projectId}/alerts/triggers', tags=["alerts", "customMetrics"]) +def get_alerts_triggers(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): + return {"data": alerts.get_predefined_values() \ + + custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} + + @app.get('/{projectId}/alerts/{alertId}', tags=["alerts"]) def get_alert(projectId: int, alertId: int, context: schemas.CurrentContext = Depends(OR_context)): return {"data": alerts.get(alertId)} @@ -1072,11 +1078,6 @@ def get_custom_metric_chart(projectId: int, data: schemas.CustomMetricChartPaylo data=data)} -@app.get('/{projectId}/custom_metrics/series', tags=["customMetrics"]) -def get_series_for_alert(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": custom_metrics.get_series_for_alert(project_id=projectId, user_id=context.user_id)} - - @app.post('/{projectId}/custom_metrics', tags=["customMetrics"]) @app.put('/{projectId}/custom_metrics', tags=["customMetrics"]) def add_custom_metric(projectId: int, data: schemas.CreateCustomMetricsSchema = Body(...), From 258ea6d9238c72b2008ff81f78f2ecd301a90afe Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 12:35:12 +0100 Subject: [PATCH 169/206] feat(api): sessions-search support Old&Flat search payload --- api/routers/core.py | 2 +- api/schemas.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/api/routers/core.py b/api/routers/core.py index 129ae9cc9..bbeb30bcd 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -108,7 +108,7 @@ def events_search(projectId: int, q: str, type: Union[schemas.FilterType, schema @app.post('/{projectId}/sessions/search2', tags=["sessions"]) -def sessions_search2(projectId: int, data: schemas.SessionsSearchPayloadSchema = Body(...), +def sessions_search2(projectId: int, data: schemas.FlatSessionsSearchPayloadSchema = Body(...), context: schemas.CurrentContext = Depends(OR_context)): data = sessions.search2_pg(data, projectId, user_id=context.user_id) return {'data': data} diff --git a/api/schemas.py b/api/schemas.py index 0dfd949ac..949ab0dff 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -458,7 +458,15 @@ class IssueType(str, Enum): js_exception = 'js_exception' -class _SessionSearchEventRaw(BaseModel): +class __MixedSearchFilter(BaseModel): + is_event: bool = Field(...) + + class Config: + alias_generator = attribute_to_camel_case + + +class _SessionSearchEventRaw(__MixedSearchFilter): + is_event: bool = Field(True, const=True) custom: Optional[List[Union[int, str]]] = Field(None, min_items=1) customOperator: Optional[MathOperator] = Field(None) key: Optional[str] = Field(None) @@ -491,7 +499,8 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): value: Union[List[_SessionSearchEventRaw], str, List[str]] = Field(...) -class _SessionSearchFilterSchema(BaseModel): +class _SessionSearchFilterSchema(__MixedSearchFilter): + is_event: bool = Field(False, const=False) custom: Optional[List[str]] = Field(None) key: Optional[str] = Field(None) value: Union[Optional[Union[IssueType, PlatformType, int, str]], @@ -536,6 +545,31 @@ class SessionsSearchPayloadSchema(BaseModel): alias_generator = attribute_to_camel_case +class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): + events: Optional[List[_SessionSearchEventSchema]] = Field([]) + filters: List[Union[_SessionSearchFilterSchema, _SessionSearchEventSchema]] = Field([]) + + @root_validator(pre=True) + def flat_to_original(cls, values): + # in case the old search body was passed + if len(values.get("events", [])) > 0: + for v in values["events"]: + v["isEvent"] = True + for v in values.get("filters", []): + v["isEvent"] = False + else: + n_filters = [] + n_events = [] + for v in values.get("filters", []): + if v["isEvent"]: + n_events.append(v) + else: + n_filters.append(v) + values["events"] = n_events + values["filters"] = n_filters + return values + + class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) order: Optional[str] = Field(default=None) From 555de84ccf0078c0926eafb566ce264040ee6f52 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 15:15:20 +0100 Subject: [PATCH 170/206] feat(api): flat custom_metrics --- api/chalicelib/core/custom_metrics.py | 20 ++++++++++++++------ api/chalicelib/utils/helper.py | 11 +++++++++++ api/schemas.py | 3 ++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/custom_metrics.py b/api/chalicelib/core/custom_metrics.py index 787b19837..10e86024c 100644 --- a/api/chalicelib/core/custom_metrics.py +++ b/api/chalicelib/core/custom_metrics.py @@ -26,7 +26,9 @@ def try_live(project_id, data: schemas.TryCustomMetricsSchema): def make_chart(project_id, user_id, metric_id, data: schemas.CustomMetricChartPayloadSchema): - metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id) + metric = get(metric_id=metric_id, project_id=project_id, user_id=user_id, flatten=False) + if metric is None: + return None metric: schemas.TryCustomMetricsSchema = schemas.TryCustomMetricsSchema.parse_obj({**data.dict(), **metric}) return try_live(project_id=project_id, data=metric) @@ -113,10 +115,10 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche u AS (UPDATE metric_series SET name=series.name, filter=series.filter, - index=series.filter.index + index=series.index FROM (VALUES {",".join([f"(%(u_series_id_{s['i']})s,%(u_index_{s['i']})s,%(u_name_{s['i']})s,%(u_filter_{s['i']})s::jsonb)" - for s in n_series])}) AS series(series_id, index, name, filter) - WHERE metric_id =%(metric_id)s AND series_id=series.series_id + for s in u_series])}) AS series(series_id, index, name, filter) + WHERE metric_series.metric_id =%(metric_id)s AND metric_series.series_id=series.series_id RETURNING 1)""") if len(d_series_ids) > 0: sub_queries.append("""\ @@ -133,7 +135,6 @@ def update(metric_id, user_id, project_id, data: schemas.UpdateCustomMetricsSche cur.execute( query ) - r = cur.fetchone() return get(metric_id=metric_id, project_id=project_id, user_id=user_id) @@ -158,6 +159,8 @@ def get_all(project_id, user_id): rows = cur.fetchall() for r in rows: r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + for s in r["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) rows = helper.list_to_camel_case(rows) return rows @@ -177,7 +180,7 @@ def delete(project_id, metric_id, user_id): return {"state": "success"} -def get(metric_id, project_id, user_id): +def get(metric_id, project_id, user_id, flatten=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -197,7 +200,12 @@ def get(metric_id, project_id, user_id): ) ) row = cur.fetchone() + if row is None: + return None row["created_at"] = TimeUTC.datetime_to_timestamp(row["created_at"]) + if flatten: + for s in row["series"]: + s["filter"] = helper.old_search_payload_to_flat(s["filter"]) return helper.dict_to_camel_case(row) diff --git a/api/chalicelib/utils/helper.py b/api/chalicelib/utils/helper.py index f8ce9fab5..6887fa5da 100644 --- a/api/chalicelib/utils/helper.py +++ b/api/chalicelib/utils/helper.py @@ -366,3 +366,14 @@ def has_smtp(): def get_edition(): return "ee" if "ee" in config("ENTERPRISE_BUILD", default="").lower() else "foss" + + +def old_search_payload_to_flat(values): + # in case the old search body was passed + if values.get("events") is not None: + for v in values["events"]: + v["isEvent"] = True + for v in values.get("filters", []): + v["isEvent"] = False + values["filters"] = values.pop("events") + values.get("filters", []) + return values diff --git a/api/schemas.py b/api/schemas.py index 949ab0dff..c5623f972 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -629,7 +629,8 @@ class MobileSignPayloadSchema(BaseModel): keys: List[str] = Field(...) -class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): +class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): +# class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): startDate: Optional[int] = Field(None) endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) From 7658d9224bf2c21904def98d730e5b9d66080415 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 15:36:27 +0100 Subject: [PATCH 171/206] feat(api): flat funnels 1/2 --- api/chalicelib/core/funnels.py | 17 +++++++++++------ api/schemas.py | 11 +++++++---- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index 835a655f4..b9db6c631 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -99,7 +99,8 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date # row["filter"]["events"] = filter_stages(row["filter"]["events"]) get_start_end_time(filter_d=row["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - counts = sessions.search2_pg(data=row["filter"], project_id=project_id, user_id=None, count_only=True) + counts = sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(row["filter"]), + project_id=project_id, user_id=None, count_only=True) row["sessionsCount"] = counts["countSessions"] row["usersCount"] = counts["countUsers"] overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) @@ -110,6 +111,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date row["criticalIssuesCount"] = overview["criticalIssuesCount"] row["missedConversions"] = 0 if len(row["stages"]) < 2 \ else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] + row["filter"] = helper.old_search_payload_to_flat(row["filter"]) return rows @@ -147,11 +149,12 @@ def delete(project_id, funnel_id, user_id): def get_sessions(project_id, funnel_id, user_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) - return sessions.search2_pg(data=f["filter"], project_id=project_id, user_id=user_id) + return sessions.search2_pg(data=schemas.SessionsSearchPayloadSchema.parse_obj(f["filter"]), project_id=project_id, + user_id=user_id) def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.FunnelSearchPayloadSchema): @@ -168,7 +171,7 @@ def get_sessions_on_the_fly(funnel_id, project_id, user_id, data: schemas.Funnel def get_top_insights(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -196,7 +199,7 @@ def get_top_insights_on_the_fly(funnel_id, user_id, project_id, data): def get_issues(project_id, user_id, funnel_id, range_value=None, start_date=None, end_date=None): - f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id) + f = get(funnel_id=funnel_id, project_id=project_id, user_id=user_id, flatten=False) if f is None: return {"errors": ["funnel not found"]} get_start_end_time(filter_d=f["filter"], range_value=range_value, start_date=start_date, end_date=end_date) @@ -224,7 +227,7 @@ def get_issues_on_the_fly(funnel_id, user_id, project_id, data): last_stage=last_stage))} -def get(funnel_id, project_id, user_id): +def get(funnel_id, project_id, user_id, flatten=True): with pg_client.PostgresClient() as cur: cur.execute( cur.mogrify( @@ -246,6 +249,8 @@ def get(funnel_id, project_id, user_id): f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) # f["filter"]["events"] = filter_stages(stages=f["filter"]["events"]) + if flatten: + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f diff --git a/api/schemas.py b/api/schemas.py index c5623f972..d66b92444 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -570,12 +570,14 @@ class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): return values -class SessionsSearchCountSchema(SessionsSearchPayloadSchema): +class SessionsSearchCountSchema(FlatSessionsSearchPayloadSchema): + # class SessionsSearchCountSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(default=None) order: Optional[str] = Field(default=None) -class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): +class FunnelSearchPayloadSchema(FlatSessionsSearchPayloadSchema): + # class FunnelSearchPayloadSchema(SessionsSearchPayloadSchema): range_value: Optional[str] = Field(None) sort: Optional[str] = Field(None) order: Optional[str] = Field(None) @@ -599,7 +601,8 @@ class UpdateFunnelSchema(FunnelSchema): is_public: Optional[bool] = Field(None) -class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): +class FunnelInsightsPayloadSchema(FlatSessionsSearchPayloadSchema): + # class FunnelInsightsPayloadSchema(SessionsSearchPayloadSchema): sort: Optional[str] = Field(None) order: Optional[str] = Field(None) @@ -630,7 +633,7 @@ class MobileSignPayloadSchema(BaseModel): class CustomMetricSeriesFilterSchema(FlatSessionsSearchPayloadSchema): -# class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): + # class CustomMetricSeriesFilterSchema(SessionsSearchPayloadSchema): startDate: Optional[int] = Field(None) endDate: Optional[int] = Field(None) sort: Optional[str] = Field(None) From 1f379435ecdd29707e8426cd852a13a6300b6484 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 22:34:32 +0530 Subject: [PATCH 172/206] chore(helm): variable naming convention Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/chalice/templates/deployment.yaml | 4 ++-- .../openreplay/charts/storage/templates/deployment.yaml | 4 ++-- scripts/helmcharts/vars.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 41df5e2f5..25ec6c387 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -69,9 +69,9 @@ spec: - name: sessions_region value: '{{ .Values.global.s3.region }}' - name: sessions_bucket - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: sourcemaps_bucket - value: '{{ .Values.global.s3.sourcemaps_bucket }}' + value: '{{ .Values.global.s3.sourcemapsBucket }}' - name: js_cache_bucket value: '{{ .Values.global.s3.assetsBucket }}' - name: EMAIL_HOST diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index 4ca3be6e2..c5ec92cf4 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -49,9 +49,9 @@ spec: - name: AWS_REGION_IOS value: '{{ .Values.global.s3.region }}' - name: S3_BUCKET_WEB - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: S3_BUCKET_IOS - value: '{{ .Values.global.s3.recordings_bucket }}' + value: '{{ .Values.global.s3.recordingsBucket }}' - name: REDIS_STRING value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - name: LICENSE_KEY diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 46d52911c..85901873f 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -43,8 +43,8 @@ global: region: "us-east-1" endpoint: "http://minio.db.svc.cluster.local:9000" assetsBucket: "sessions-assets" - recordings_bucket: "mobs" - sourcemaps_bucket: "sourcemaps" + recordingsBucket: "mobs" + sourcemapsBucket: "sourcemaps" # if you're using one node installation, where # you're using local s3, make sure these variables # are same as minio.global.minio.accesskey and secretKey From 76f925beed0affadb1df38bc1fb6ec168da1c44c Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Fri, 21 Jan 2022 23:21:42 +0530 Subject: [PATCH 173/206] chore(helm): example for db upgrade Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 85901873f..a19ab96ae 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -8,6 +8,13 @@ postgresql: &postgres postgresqlPort: "5432" postgresqlUser: "postgres" postgresqlDatabase: "postgres" + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 3000Mi + # cpu: 2 clickhouse: {} # For enterpriseEdition @@ -91,6 +98,8 @@ chalice: ## Changes to nginx # # nginx-ingress: +# sslKey: site.crt +# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; From ef8eb2e9940b74a1b28686f1291e5a2aaacaea8e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:46:02 +0100 Subject: [PATCH 174/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index a19ab96ae..5fd626d48 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -90,10 +90,14 @@ chalice: # For example: # -# alerts: +# http: # resources: # limits: -# cpu: 1 +# cpu: 1024m +# memory: 4096Mi +# requests: +# cpu: 512m +# memory: 2056Mi ## Changes to nginx # From 29a37918f571a6e8d3b2dbd0c2e8c81f82a87c06 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:46:17 +0100 Subject: [PATCH 175/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 5fd626d48..9ad685642 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -108,20 +108,3 @@ chalice: # # Redirecting http to https # return 301 https://$host$request_uri; # -### Change the ssl certificates -### Public certificate ( content from site.crt, mind the indentation ) -# ssl_certificate: |- -# -----BEGIN CERTIFICATE----- -# MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL -# BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 -# NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh -# -----END CERTIFICATE----- -# -### Private certificate ( content from site.key, mind the indentation. ) -# ssl_privatekey: |- -# -----BEGIN PRIVATE KEY----- -# TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T -# e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP -# 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 -# 7qINKoGovqXB1oAdopIl1z64e7MWVE4= -# -----END PRIVATE KEY----- From 56c4308b20a96b66132f6252640ab17f680af55c Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Fri, 21 Jan 2022 18:02:36 +0100 Subject: [PATCH 176/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 9ad685642..62482dd50 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -74,6 +74,8 @@ global: chalice: env: jwt_secret: "SetARandomStringHere" + # captcha_server: '' + # captcha_key: '' # SAML2_MD_URL: '' # idp_entityId: '' # idp_sso_url: '' From e474ecd0bf4c5619b4c566a1ba0e51bba066fb18 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 22 Jan 2022 21:12:06 +0530 Subject: [PATCH 177/206] chore(helm): read nginx ssl from file Signed-off-by: Rajesh Rajendran --- .../charts/nginx-ingress/files/site.crt | 1 + .../charts/nginx-ingress/files/site.key | 1 + .../nginx-ingress/templates/secrets.yaml | 6 +- .../charts/nginx-ingress/values.yaml | 87 ------------------- scripts/helmcharts/openreplay/files/site.crt | 30 +++++++ scripts/helmcharts/openreplay/files/site.key | 52 +++++++++++ scripts/helmcharts/vars.yaml | 2 - 7 files changed, 87 insertions(+), 92 deletions(-) create mode 120000 scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt create mode 120000 scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key create mode 100644 scripts/helmcharts/openreplay/files/site.crt create mode 100644 scripts/helmcharts/openreplay/files/site.key diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt new file mode 120000 index 000000000..12e23824a --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.crt @@ -0,0 +1 @@ +../../../files/site.crt \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key new file mode 120000 index 000000000..3805a27d1 --- /dev/null +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/files/site.key @@ -0,0 +1 @@ +../../../files/site.key \ No newline at end of file diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml index e5e4d7dd9..91b7cc09c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/secrets.yaml @@ -4,6 +4,6 @@ kind: Secret metadata: name: ssl data: - ca.crt: "" - site.crt: "{{ .Values.ssl_certificate | b64enc }}" - site.key: "{{ .Values.ssl_privatekey | b64enc }}" + ca.crt: '' + site.crt: '{{ .Files.Get "files/site.crt" | b64enc }}' + site.key: '{{ .Files.Get "files/site.key" | b64enc }}' diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml index 8c595aafa..821ad9e3c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/values.yaml @@ -84,90 +84,3 @@ nodeSelector: {} tolerations: [] affinity: {} - - -ssl_certificate: |- - -----BEGIN CERTIFICATE----- - MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL - BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 - NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh - bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ - LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w - hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds - QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX - sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o - immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l - fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ - ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ - OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u - tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p - CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA - YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 - z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs - MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z - qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA - dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p - 0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 - 5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh - 0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s - /JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP - XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc - HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG - T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi - qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S - w7yMPvE+TIh6+HwDZc2jNLkv/8tY - -----END CERTIFICATE----- - -ssl_privatekey: |- - -----BEGIN PRIVATE KEY----- - MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t - K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE - BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB - OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew - 6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK - aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 - hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK - lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 - VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 - RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK - Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh - QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe - K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s - dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 - CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh - ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl - 5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw - TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk - x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c - Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g - DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj - OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm - 5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p - /CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA - 0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 - tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E - ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy - HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 - IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA - PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q - p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO - GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw - 1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ - cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d - vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ - n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ - Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x - OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 - Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP - kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB - De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr - TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T - e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP - 3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 - WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa - 8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb - esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn - PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 - PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 - 7qINKoGovqXB1oAdopIl1z64e7MWVE4= - -----END PRIVATE KEY----- diff --git a/scripts/helmcharts/openreplay/files/site.crt b/scripts/helmcharts/openreplay/files/site.crt new file mode 100644 index 000000000..90f6a68c2 --- /dev/null +++ b/scripts/helmcharts/openreplay/files/site.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFITCCAwmgAwIBAgIUQ8hQoDbW3Z4DxRVjIYlIlbEHp/8wDQYJKoZIhvcNAQEL +BQAwIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2NhbC5ob3N0MB4XDTIxMTIyMjA3 +NDIxOVoXDTIyMTIyMjA3NDIxOVowIDEeMBwGA1UEAwwVb3BlbnJlcGxheS5sb2Nh +bC5ob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyXTX6RwqNVM+ +LSvc5TkuBnxlw1sHxtkkojwpbwavr6ccSdtoYB7KYwcufh0zz3LaSDgPNqStOf6w +hAWV830bxvOvU6yJ7MgP8/htfY1KWIoNS6ducoct4VhgshWXWwQtrtWZJku+cyds +QTkr2BziSX+Y7/1rALKbOU4CIRCKtJ2jeaI+c4kcXXB+ARauDlqB7+CS4B+wjlfX +sOoC2bWgZOxyZnHolb3hKMLfBswLwYq0DRjjNMDqX8xS6V1AgoTrCxl1DqPLw47o +immbSKZ4voot60cSBYVK4qOX5Nqw5RmqwELb9Ib4QPVCt9HjbYQp77EcOonkgE4l +fYabvvOeM/U6vdtZhI2CJg0tkytuJ4+Hb7i7nRK2SRMppmtP7yDDXpMGoAXK2bVZ +ipZBRct0onxLifH5vdrUNbOlXItjWLQMfiHlDeG48kbXbKaJPv3tRvU0Gix2X8SJ +OlRNezNNz8pce0Bbgx3YoQhrRTad4CC6cIpRjgTt/pww3BoF7jDLl6RNI1cXfy4u +tkSlMqAQV6x0aig9Ldg1VFM2oCaEyvzx0BWDm/jmbZcyVizlb+uQQ/huNSJXT++p +CmPNG7rP6eYNTh7+7DDWvKBQQFWOPaVfwvrhzvb7q2B2Bmc33bDXeRuF4MJE6syA +YUCV2Ztw65uI864PRDIKO4ru1UQgx5sCAwEAAaNTMFEwHQYDVR0OBBYEFIdNQGn2 +z3xmJfExKAUeohFnLjzsMB8GA1UdIwQYMBaAFIdNQGn2z3xmJfExKAUeohFnLjzs +MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAJvkSX+RoYGOcU0z +qEWlAN6Jy0CzLsTp/VoSZY9gd+hLFd/lK7SjXhAnNyEELpeanGs04ytJpXnSH1bA +dGs0UeWYalOxrHSN4ln5SNzWVE8kODM1zcyNePllI/PVcXLmujQz5wNbNoC5Qt8p +0RoZ2wInmyh2wTQcflIPUtncsw84ozVVuebmc6jiuPxnxdTAXeYOwKUF25t8rSp6 +5n23F0GP8Ypu7vjT7N2RpUe0zkutaij+uISBHZw50ohrelPlV4V9qhp6MV+h9xuh +0z8OEyq2vK4KNn96A97mSRuqqt6Ajb2MHdErTr6fgj5/CtSD337oIK3froRmID8s +/JXADsNnBEqQBfcM6gSaw1M/fHDPNZzwVv6yAN+bKrI+KEmKJD31Tm2G55oPvLTP +XZdmVIAqxIu89v/GOJ2J29vC+h9pTjTze31DFg0niwLcr1aNawiC2d4n2wdDwKwc +HnCnflELyYcn4KgvpLNz5wEKEHTAQ3JF5VIel1/uqYN9cosw1vjRskPK/g3nIEPG +T247naj+JbW244P0jxb57VWiD/7IJ4ZErA1KrvqR/y1NnGxrgXoRjwmhCv/4YIYi +qgnvF7IkwGozdoLPiBMmvjNq/AmVLrfZNPxZjHL3nIW+PBEeBD/lkH36mcakg/1S +w7yMPvE+TIh6+HwDZc2jNLkv/8tY +-----END CERTIFICATE----- diff --git a/scripts/helmcharts/openreplay/files/site.key b/scripts/helmcharts/openreplay/files/site.key new file mode 100644 index 000000000..641fd1b7b --- /dev/null +++ b/scripts/helmcharts/openreplay/files/site.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDJdNfpHCo1Uz4t +K9zlOS4GfGXDWwfG2SSiPClvBq+vpxxJ22hgHspjBy5+HTPPctpIOA82pK05/rCE +BZXzfRvG869TrInsyA/z+G19jUpYig1Lp25yhy3hWGCyFZdbBC2u1ZkmS75zJ2xB +OSvYHOJJf5jv/WsAsps5TgIhEIq0naN5oj5ziRxdcH4BFq4OWoHv4JLgH7COV9ew +6gLZtaBk7HJmceiVveEowt8GzAvBirQNGOM0wOpfzFLpXUCChOsLGXUOo8vDjuiK +aZtIpni+ii3rRxIFhUrio5fk2rDlGarAQtv0hvhA9UK30eNthCnvsRw6ieSATiV9 +hpu+854z9Tq921mEjYImDS2TK24nj4dvuLudErZJEymma0/vIMNekwagBcrZtVmK +lkFFy3SifEuJ8fm92tQ1s6Vci2NYtAx+IeUN4bjyRtdspok+/e1G9TQaLHZfxIk6 +VE17M03Pylx7QFuDHdihCGtFNp3gILpwilGOBO3+nDDcGgXuMMuXpE0jVxd/Li62 +RKUyoBBXrHRqKD0t2DVUUzagJoTK/PHQFYOb+OZtlzJWLOVv65BD+G41IldP76kK +Y80bus/p5g1OHv7sMNa8oFBAVY49pV/C+uHO9vurYHYGZzfdsNd5G4XgwkTqzIBh +QJXZm3Drm4jzrg9EMgo7iu7VRCDHmwIDAQABAoICAQCebjlupiu7jB+Vvq0VyAYe +K66MGAbhpttcixu6qPN5nF5u5xIKpaxcfMVfgO/B8X0g1pWAT7m7pkSDTzFCL92s +dPApScOeZyfEolbZKkiRoOAb4yzE/PJkCfDhnIFPntWebXTn3SGFxjcohCGq7+w2 +CRbphc6k2dGhG2wpPK0YpfBuM94RVn7sLQ+rI3724s7VKzPW9pUPHJ4QD7j2JhRh +ymGdl29mc9GjEL38xnNoXgCDXFMypZSsii+aPzAAdS+zpu2b+czBmp3eXHc2h1Tl +5B2Arn/Jv63I1wcZf7MmOS1DzlDU2WBbFYbGsVW+RvYD/rFIiDEfhlWNhlLttQFw +TJ9xk+EePK9VQuWzN5tG1lEjGcNWtPLUp3IxZTqaei5lWu6zyA6HVsxjyArzmfNk +x0fRpZU+VZYzbkgj0ROq3wg7QEEMQ8SPo9vvLF1ZNnndzs/ziPA1CodUuSwa6B2c +Zeref4s0B//q3U1SDQE08OD9iuZODwtkO4wQtW2DP33gC6VIts94jg87z8SRDp2g +DcT3D8ZhV5B2VPelluQZ/scWKGWKAvPVRjq51EiMeZtFBVyM6+o0xW2+MxxZdjbj +OWexc+dw8QfwIlFRm0v8Tfvljk1prqYEMLV4s9JD8up5X1h3Yg5uAsQpdZ+1JkGm +5UvvQQVQgxkC1NFXxqYyQQKCAQEA95r3oYm+bnQXHXX4dBQO98+XwOOfnhMk815p +/CAuxCzbPNafjqyAxRgmp5D/IkdLzbitDL3uo73ot1RwB4CEN/Ovp3a/+CklBnnA +0bKAtsGE2XWaqFeguVIy25WEWKaTxKGX0g6KHkOvGt0DNo4wUJUk+2sAqIvXU1Q6 +tUbd+8YRYxO7i6+92K7kxoZega6qiA/L3akZ2uTzFf+IskfqmDUoF2ZaEOFluG8E +ASX3KoVFfraV3DBEN0ionvfpaRIidr2IsuC848zHFBtAXA0mL55BCuf++HmAZnpy +HFN7owVVgqbEw+GGbNdRLt5zV00DmX/sHsIZU/gCLRPsfPUAqQKCAQEA0ElWWiS4 +IA91lWbzCwswFrHvjpcogh67fNd9kJCcFCUHabev7SSrIEcjqDP2m6HPvp/FwxYA +PEo1/vDZ884v9roft2J13OvpvXoqtRZLGo1E76sECBrcto4nhCiTeRQg9uRpHG+Q +p77QC/4eRBLGykFRJET6913x7JzpjAO0QLLLzilj1yBkbF5U01Up5KbIuNeXNvEO +GVGpbryIXxwR6Qhyv7C54xpjRdu9EOT1frRqdIs0qOGafnLXWAXKfvWUzz1wSiiw +1p7xqYZrawXAr7XEkGA2aeqt/iqo2X2G9oYA0apJVwfR4WhuS2hPkSy405bsrGzZ +cjMs9bnJSYP8owKCAQEAxCTSvfisDjuQhBoL84hgQxcEFB09OK/ZuaC1PLER2v3d +vtgWFaO5bmivVlaahcEM367IByv+e1/ZlkEhbg/0rY4xO+vqLuAJIJQalwNcy2mJ +n+p11Z11CNmAyEotSTzMGhwYdKJn74mWkSU7gmApDezYGwKsxtfgf3Zd+3RkLSq+ +Y0oia4mQTrJdMJcJDpobJSW+TZ3DiY+MsYR3+SLXSDPzynWeK3kiZ3QqK+6zWc+x +OavSE1d48oJwcV3aXQ2sl3uVan51o894dQkRdtpDwb0PsWAOry8w8/1Tn/TSIFX9 +Yz5Q6Qsivd3jxckafbHYhCS+G6+O+OGid6ssz+AV4QKCAQAqK78ND0QsUZT4A9kP +kltRLQOYtiggeEJzm1mz7GN9fKXMlMFM3VC8f0rL4oF6rz9VlBnBTvILQuc9z9wB +De0OIk8LnSbJ7QXtNA/zjCj2nkWn1NNDJNUtLQj5LBH3wMiP1F0nwbrjC7Ipy3Cr +TbXr+1+HXWQGs4Go63gpvhI/yzOScTTiuI88lbjM9QA/aDlZm2TlXdcB71PDtO5T +e2Zw7SH2h7yLK6uP2FamVgUSe0rWf9zQmKTkFzJcgwelvuk7MHBMw4JSYeoB7dJP +3+FMchvzM1exCC/kNxTqvAyYWzdNPBIPSekHn1I9eEgr14cwZ+1RV9SK16uxsMT9 +WnjLAoIBADKutRKB8nH+wD3sa4cP782QNbkDqJCcb3rPntnCWI/jA2TeY/wAvrXa +8yFtSSeYSwN9Wr+UosSkQ+OQSO0WmT2NrxdkH8jK8kYnzYkJ9+EFE2YpMN2UosSb +esQ9oEMnivBMNv8DnB4IuO8LjTj1rhqcBmWJH1zvDi1Ur+/uAb+6XLm0Dp/59/Rn +PSlLQmFraq6mrUkKTU40zyT6eK8AvIn/+sXAF1Xb9Vnm8Ndl+gZ4imzjcCubbq+6 +PqvLjFJNGyya6b3MX4RSxVGfkIf5f6bcSSZ0zzSB3qLbCKS+JawwR1WF2rJp6Hj5 +7qINKoGovqXB1oAdopIl1z64e7MWVE4= +-----END PRIVATE KEY----- diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 62482dd50..9fbd08c98 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -104,8 +104,6 @@ chalice: ## Changes to nginx # # nginx-ingress: -# sslKey: site.crt -# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; From 7b4adc92bd57c3ef3ffda3a4c8edad7390ad7af2 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Sat, 22 Jan 2022 21:31:12 +0530 Subject: [PATCH 178/206] chore(helm): create vars_template for variable migration --- scripts/helmcharts/vars_template.yaml | 111 ++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 scripts/helmcharts/vars_template.yaml diff --git a/scripts/helmcharts/vars_template.yaml b/scripts/helmcharts/vars_template.yaml new file mode 100644 index 000000000..84ce36ee6 --- /dev/null +++ b/scripts/helmcharts/vars_template.yaml @@ -0,0 +1,111 @@ +fromVersion: "{{ openreplay_version }}" +# Databases specific variables +postgresql: &postgres + # For generating passwords + # `openssl rand -hex 20` + postgresqlPassword: "{{ postgres_db_password }}" + postgresqlHost: "{{ postgres_endpoint }}" + postgresqlPort: "5432" + postgresqlUser: "{{ postgres_db_user }}" + postgresqlDatabase: "{{ postgres_db_name }}" + # resources: + # requests: + # memory: 256Mi + # cpu: 250m + # limits: + # memory: 3000Mi + # cpu: 2 + +clickhouse: {} + # For enterpriseEdition + # enabled: true + +kafka: &kafka + # For enterpriseEdition + # enabled: true + + kafkaHost: "{{ kafka_endpoint }}" + kafkaPort: "{{ kafka_endpoint.split(':')[-1] }}" + kafkaUseSsl: "{{ kafka_ssl }}" + +redis: &redis + # For enterpriseEdition + # enabled: false + redisHost: "{{ redis_endpoint }}" + redisPort: "{{ redis_endpoint.split(':')[-1] }}" + +minio: + global: + minio: + # For generating passwords + # `openssl rand -hex 20` + accessKey: "{{ minio_access_key }}" + secretKey: "{{ minio_secret_key }}" + +# Application specific variables +global: + postgresql: *postgres + kafka: *kafka + redis: *redis + s3: + region: "us-east-1" + endpoint: "http://minio.db.svc.cluster.local:9000" + assetsBucket: "sessions-assets" + recordingsBucket: "mobs" + sourcemapsBucket: "sourcemaps" + # if you're using one node installation, where + # you're using local s3, make sure these variables + # are same as minio.global.minio.accesskey and secretKey + accessKey: "{{ minio_access_key }}" + secretKey: "{{ minio_secret_key }}" + email: + emailHost: '{{ email_host }}' + emailPort: '{{ email_port }}' + emailUser: '{{ email_user }}' + emailPassword: '{{ email_password }}' + emailUseTls: '{{ email_use_tls }}' + emailUseSsl: '{{ email_use_ssl }}' + emailSslKey: '{{ email_ssl_key }}' + emailSslCert: '{{ email_ssl_cert }}' + emailFrom: '{{ email_from }}' + + enterpriseEditionLicense: "{{ enterprise_edition_license }}" + domainName: "{{ domain_name }}" + +chalice: + env: + jwt_secret: "{{ jwt_secret_key }}" + # captcha_server: '' + # captcha_key: '' + # SAML2_MD_URL: '' + # idp_entityId: '' + # idp_sso_url: '' + # idp_x509cert: '' + # idp_sls_url: '' + # idp_name: '' + # idp_tenantKey: '' + + +# If you want to override something +# chartname: +# filedFrom chart/Values.yaml: +# key: value + +# For example: +# +# http: +# resources: +# limits: +# cpu: 1024m +# memory: 4096Mi +# requests: +# cpu: 512m +# memory: 2056Mi + +## Changes to nginx +# +# nginx-ingress: +# customServerConfigs: | +# # Redirecting http to https +# return 301 https://$host$request_uri; +# From a2a34e90f6bc4b1a0ae805b41d51f9fc2dc43e0f Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 18:20:16 +0100 Subject: [PATCH 179/206] feat(api): flat funnels --- api/chalicelib/core/funnels.py | 3 ++- api/chalicelib/core/significance.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/funnels.py b/api/chalicelib/core/funnels.py index b9db6c631..c33bed586 100644 --- a/api/chalicelib/core/funnels.py +++ b/api/chalicelib/core/funnels.py @@ -103,6 +103,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date project_id=project_id, user_id=None, count_only=True) row["sessionsCount"] = counts["countSessions"] row["usersCount"] = counts["countUsers"] + filter_clone = dict(row["filter"]) overview = significance.get_overview(filter_d=row["filter"], project_id=project_id) row["stages"] = overview["stages"] row.pop("filter") @@ -111,7 +112,7 @@ def get_by_user(project_id, user_id, range_value=None, start_date=None, end_date row["criticalIssuesCount"] = overview["criticalIssuesCount"] row["missedConversions"] = 0 if len(row["stages"]) < 2 \ else row["stages"][0]["sessionsCount"] - row["stages"][-1]["sessionsCount"] - row["filter"] = helper.old_search_payload_to_flat(row["filter"]) + row["filter"] = helper.old_search_payload_to_flat(filter_clone) return rows diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index d81378ddb..f2261ce59 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -2,7 +2,7 @@ __author__ = "AZNAUROV David" __maintainer__ = "KRAIEM Taha Yassine" import schemas -from chalicelib.core import events, sessions_metas, metadata, sessions +from chalicelib.core import events, metadata, sessions from chalicelib.utils import dev """ @@ -617,6 +617,15 @@ def get_overview(filter_d, project_id, first_stage=None, last_stage=None): # The result of the multi-stage query rows = get_stages_and_events(filter_d=filter_d, project_id=project_id) if len(rows) == 0: + # PS: not sure what to return if rows are empty + output["stages"] = [{ + "type": stages[0]["type"], + "value": stages[0]["value"], + "sessionsCount": None, + "dropPercentage": None, + "usersCount": None + }] + output['criticalIssuesCount'] = 0 return output # Obtain the first part of the output stages_list = get_stages(stages, rows) From b8caca0223c172288c42589a4d0f8ef4f5c8456b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sat, 22 Jan 2022 19:56:32 +0100 Subject: [PATCH 180/206] feat(api): flat saved_search --- api/chalicelib/core/saved_search.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/chalicelib/core/saved_search.py b/api/chalicelib/core/saved_search.py index dfa9a1dcf..732fc1596 100644 --- a/api/chalicelib/core/saved_search.py +++ b/api/chalicelib/core/saved_search.py @@ -18,6 +18,7 @@ def create(project_id, user_id, data: schemas.SavedSearchSchema): ) r = cur.fetchone() r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["filter"] = helper.old_search_payload_to_flat(r["filter"]) r = helper.dict_to_camel_case(r) return {"data": r} @@ -40,6 +41,7 @@ def update(search_id, project_id, user_id, data: schemas.SavedSearchSchema): ) r = cur.fetchone() r["created_at"] = TimeUTC.datetime_to_timestamp(r["created_at"]) + r["filter"] = helper.old_search_payload_to_flat(r["filter"]) r = helper.dict_to_camel_case(r) # r["filter"]["startDate"], r["filter"]["endDate"] = TimeUTC.get_start_end_from_range(r["filter"]["rangeValue"]) return r @@ -74,6 +76,8 @@ def get_all(project_id, user_id, details=False): rows = helper.list_to_camel_case(rows) for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) + if details: + row["filter"] = helper.old_search_payload_to_flat(row["filter"]) return rows @@ -112,4 +116,5 @@ def get(search_id, project_id, user_id): return None f["createdAt"] = TimeUTC.datetime_to_timestamp(f["createdAt"]) + f["filter"] = helper.old_search_payload_to_flat(f["filter"]) return f From 1b62d4dd7497dee729da0ebcd364aecaeb5fd383 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 23 Jan 2022 11:46:43 +0100 Subject: [PATCH 181/206] Delete README.md --- scripts/helmcharts/README.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 scripts/helmcharts/README.md diff --git a/scripts/helmcharts/README.md b/scripts/helmcharts/README.md deleted file mode 100644 index b54ca9652..000000000 --- a/scripts/helmcharts/README.md +++ /dev/null @@ -1,7 +0,0 @@ -- Initialize databases - - we've to pass the --wait flag, else the db installation won't be complete. and it'll break the db init. - -## Installation -helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic - -helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./values.yaml --atomic From 5e82278b4d6fd4a68aba88ba16e6cb48510b424e Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 23 Jan 2022 11:56:47 +0100 Subject: [PATCH 182/206] Update vars.yaml --- scripts/helmcharts/vars.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 9fbd08c98..d6b403408 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -89,9 +89,8 @@ chalice: # chartname: # filedFrom chart/Values.yaml: # key: value - -# For example: # +# For example (http): # http: # resources: # limits: @@ -104,6 +103,10 @@ chalice: ## Changes to nginx # # nginx-ingress: +# # Key and certificate files must be named site.key and site.crt +# # and copied to ../openreplay/files/ +# sslKey: site.key +# sslCert: site.crt # customServerConfigs: | # # Redirecting http to https # return 301 https://$host$request_uri; From 7196b819791f138f7b37434a45727c12e425a601 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 17:52:58 +0100 Subject: [PATCH 183/206] feat(tracker-assist): 3.4.16: improve scroll, ignore Fetch, peerjs logs --- tracker/tracker-assist/package.json | 2 +- tracker/tracker-assist/src/BufferingConnection.ts | 6 +++++- tracker/tracker-assist/src/Mouse.ts | 5 +++-- tracker/tracker-assist/src/index.ts | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tracker/tracker-assist/package.json b/tracker/tracker-assist/package.json index 506f05e2d..4d327410b 100644 --- a/tracker/tracker-assist/package.json +++ b/tracker/tracker-assist/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker-assist", "description": "Tracker plugin for screen assistance through the WebRTC", - "version": "3.4.13", + "version": "3.4.16", "keywords": [ "WebRTC", "assistance", diff --git a/tracker/tracker-assist/src/BufferingConnection.ts b/tracker/tracker-assist/src/BufferingConnection.ts index e90970c21..5fb3b7349 100644 --- a/tracker/tracker-assist/src/BufferingConnection.ts +++ b/tracker/tracker-assist/src/BufferingConnection.ts @@ -6,12 +6,13 @@ interface Message { } // 16kb should be max according to specification +// 64kb chrome const crOrFf: boolean = typeof navigator !== "undefined" && (navigator.userAgent.indexOf("Chrom") !== -1 || // Chrome && Chromium navigator.userAgent.indexOf("Firefox") !== -1); -const MESSAGES_PER_SEND = crOrFf ? 500 : 100 +const MESSAGES_PER_SEND = crOrFf ? 200 : 50 // Bffering required in case of webRTC export default class BufferingConnection { @@ -34,7 +35,10 @@ export default class BufferingConnection { send(messages: Message[]) { if (!this.conn.open) { return; } let i = 0; + //@ts-ignore + messages=messages.filter(m => m._id !== 39) while (i < messages.length) { + this.buffer.push(messages.slice(i, i+=this.msgsPerSend)) } if (!this.buffering) { diff --git a/tracker/tracker-assist/src/Mouse.ts b/tracker/tracker-assist/src/Mouse.ts index d2c89cfe6..b183413bd 100644 --- a/tracker/tracker-assist/src/Mouse.ts +++ b/tracker/tracker-assist/src/Mouse.ts @@ -47,11 +47,12 @@ export default class Mouse { } } - private readonly pScrEl = document.scrollingElement || document.documentElement + private readonly pScrEl = document.scrollingElement || document.documentElement // Is it always correct private lastScrEl: Element | "window" | null = null private resetLastScrEl = () => { this.lastScrEl = null } private handleWScroll = e => { - if (e.target !== this.lastScrEl) { + if (e.target !== this.lastScrEl && + this.lastScrEl !== "window") { this.resetLastScrEl() } } diff --git a/tracker/tracker-assist/src/index.ts b/tracker/tracker-assist/src/index.ts index 74514378a..d2067ff91 100644 --- a/tracker/tracker-assist/src/index.ts +++ b/tracker/tracker-assist/src/index.ts @@ -27,7 +27,7 @@ enum CallingState { }; //@ts-ignore peerjs hack for webpack5 (?!) TODO: ES/node modules; -//Peer = Peer.default || Peer; +Peer = Peer.default || Peer; // type IncomeMessages = // "call_end" | @@ -86,7 +86,7 @@ export default function(opts?: Partial) { host: app.getHost(), path: '/assist', port: location.protocol === 'http:' && appOptions.__DISABLE_SECURE_MODE ? 80 : 443, - //debug: // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. + debug: appOptions.__debug_log ? 2 : 0, // 0 Print nothing //1 Prints only errors. / 2 Prints errors and warnings. / 3 Prints all logs. } if (options.config) { _opt['config'] = options.config From b3fba224cc5d248ceaaf6364dc8ae1e7e0285f65 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 18:28:22 +0100 Subject: [PATCH 184/206] frat(tracker): 3.4.16: observer for shRoot, topObserver & sanitizer refactor; 3.4.17:autoResetOnWindowOpen; userID onStart option; restart fix --- tracker/tracker/package-lock.json | 2 +- tracker/tracker/package.json | 2 +- tracker/tracker/src/main/app/context.ts | 72 +++ tracker/tracker/src/main/app/index.ts | 246 +++++---- tracker/tracker/src/main/app/observer.ts | 484 ------------------ .../src/main/app/observer/iframe_observer.ts | 19 + .../tracker/src/main/app/observer/observer.ts | 353 +++++++++++++ .../main/app/observer/shadow_root_observer.ts | 18 + .../src/main/app/observer/top_observer.ts | 98 ++++ tracker/tracker/src/main/app/sanitizer.ts | 66 +++ tracker/tracker/src/main/index.ts | 65 ++- tracker/tracker/src/main/modules/exception.ts | 8 +- tracker/tracker/src/main/modules/img.ts | 19 +- tracker/tracker/src/main/modules/input.ts | 20 +- tracker/tracker/src/main/modules/mouse.ts | 2 +- tracker/tracker/src/webworker/index.ts | 1 + 16 files changed, 853 insertions(+), 622 deletions(-) create mode 100644 tracker/tracker/src/main/app/context.ts delete mode 100644 tracker/tracker/src/main/app/observer.ts create mode 100644 tracker/tracker/src/main/app/observer/iframe_observer.ts create mode 100644 tracker/tracker/src/main/app/observer/observer.ts create mode 100644 tracker/tracker/src/main/app/observer/shadow_root_observer.ts create mode 100644 tracker/tracker/src/main/app/observer/top_observer.ts create mode 100644 tracker/tracker/src/main/app/sanitizer.ts diff --git a/tracker/tracker/package-lock.json b/tracker/tracker/package-lock.json index 287203b30..6dcbc4e81 100644 --- a/tracker/tracker/package-lock.json +++ b/tracker/tracker/package-lock.json @@ -1,6 +1,6 @@ { "name": "@openreplay/tracker", - "version": "3.4.7", + "version": "3.4.12", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/tracker/tracker/package.json b/tracker/tracker/package.json index a2dbc60f1..16b10b8f4 100644 --- a/tracker/tracker/package.json +++ b/tracker/tracker/package.json @@ -1,7 +1,7 @@ { "name": "@openreplay/tracker", "description": "The OpenReplay tracker main package", - "version": "3.4.12", + "version": "3.4.17", "keywords": [ "logging", "replay" diff --git a/tracker/tracker/src/main/app/context.ts b/tracker/tracker/src/main/app/context.ts new file mode 100644 index 000000000..aa9a5dfb3 --- /dev/null +++ b/tracker/tracker/src/main/app/context.ts @@ -0,0 +1,72 @@ +// TODO: global type +export interface Window extends globalThis.Window { + HTMLInputElement: typeof HTMLInputElement, + HTMLLinkElement: typeof HTMLLinkElement, + HTMLStyleElement: typeof HTMLStyleElement, + SVGStyleElement: typeof SVGStyleElement, + HTMLIFrameElement: typeof HTMLIFrameElement, + Text: typeof Text, + Element: typeof Element, + ShadowRoot: typeof ShadowRoot, + //parent: Window, +} + +type WindowConstructor = + Document | + Element | + Text | + ShadowRoot | + HTMLInputElement | + HTMLLinkElement | + HTMLStyleElement | + HTMLIFrameElement + +// type ConstructorNames = +// 'Element' | +// 'Text' | +// 'HTMLInputElement' | +// 'HTMLLinkElement' | +// 'HTMLStyleElement' | +// 'HTMLIFrameElement' +type Constructor = { new (...args: any[]): T , name: string }; + + // TODO: we need a type expert here so we won't have to ignore the lines + // TODO: use it everywhere (static function; export from which file? <-- global Window typing required) +export function isInstance(node: Node, constr: Constructor): node is T { + const doc = node.ownerDocument; + if (!doc) { // null if Document + return constr.name === 'Document'; + } + let context: Window = + // @ts-ignore (for EI, Safary) + doc.parentWindow || + doc.defaultView; // TODO: smart global typing for Window object + while(context.parent && context.parent !== context) { + // @ts-ignore + if (node instanceof context[constr.name]) { + return true + } + // @ts-ignore + context = context.parent + } + // @ts-ignore + return node instanceof context[constr.name] +} + +export function inDocument(node: Node): boolean { + const doc = node.ownerDocument + if (!doc) { return false } + if (doc.contains(node)) { return true } + let context: Window = + // @ts-ignore (for EI, Safary) + doc.parentWindow || + doc.defaultView; + while(context.parent && context.parent !== context) { + if (context.document.contains(node)) { + return true + } + // @ts-ignore + context = context.parent + } + return false; +} diff --git a/tracker/tracker/src/main/app/index.ts b/tracker/tracker/src/main/app/index.ts index 54fe9050f..b02d15f91 100644 --- a/tracker/tracker/src/main/app/index.ts +++ b/tracker/tracker/src/main/app/index.ts @@ -2,12 +2,15 @@ import { timestamp, log, warn } from "../utils.js"; import { Timestamp, PageClose } from "../../messages/index.js"; import Message from "../../messages/message.js"; import Nodes from "./nodes.js"; -import Observer from "./observer.js"; +import Observer from "./observer/top_observer.js"; +import Sanitizer from "./sanitizer.js"; import Ticker from "./ticker.js"; import { deviceMemory, jsHeapSizeLimit } from "../modules/performance.js"; -import type { Options as ObserverOptions } from "./observer.js"; +import type { Options as ObserverOptions } from "./observer/top_observer.js"; +import type { Options as SanitizerOptions } from "./sanitizer.js"; + import type { Options as WebworkerOptions, WorkerMessageData } from "../../messages/webworker.js"; @@ -17,11 +20,17 @@ export interface OnStartInfo { userUUID: string, } -export type Options = { +export interface StartOptions { + userID?: string, + forceNew: boolean, +} + +type AppOptions = { revID: string; node_id: string; session_token_key: string; session_pageno_key: string; + session_reset_key: string; local_uuid_key: string; ingestPoint: string; resourceBaseHref: string | null, // resourceHref? @@ -30,7 +39,9 @@ export type Options = { __debug_report_edp: string | null; __debug_log: boolean; onStart?: (info: OnStartInfo) => void; -} & ObserverOptions & WebworkerOptions; +} & WebworkerOptions; + +export type Options = AppOptions & ObserverOptions & SanitizerOptions type Callback = () => void; type CommitCallback = (messages: Array) => void; @@ -43,21 +54,23 @@ export default class App { readonly nodes: Nodes; readonly ticker: Ticker; readonly projectKey: string; + readonly sanitizer: Sanitizer; private readonly messages: Array = []; - /*private*/ readonly observer: Observer; // temp, for fast security fix. TODO: separate security/obscure module with nodeCallback that incapsulates `textMasked` functionality from Observer + private readonly observer: Observer; private readonly startCallbacks: Array = []; private readonly stopCallbacks: Array = []; private readonly commitCallbacks: Array = []; - private readonly options: Options; + private readonly options: AppOptions; private readonly revID: string; private _sessionID: string | null = null; + private _userID: string | undefined; private isActive = false; private version = 'TRACKER_VERSION'; private readonly worker?: Worker; constructor( projectKey: string, sessionToken: string | null | undefined, - opts: Partial, + options: Partial, ) { this.projectKey = projectKey; this.options = Object.assign( @@ -66,24 +79,23 @@ export default class App { node_id: '__openreplay_id', session_token_key: '__openreplay_token', session_pageno_key: '__openreplay_pageno', + session_reset_key: '__openreplay_reset', local_uuid_key: '__openreplay_uuid', ingestPoint: DEFAULT_INGEST_POINT, resourceBaseHref: null, __is_snippet: false, __debug_report_edp: null, __debug_log: false, - obscureTextEmails: true, - obscureTextNumbers: false, - captureIFrames: false, }, - opts, + options, ); if (sessionToken != null) { sessionStorage.setItem(this.options.session_token_key, sessionToken); } this.revID = this.options.revID; + this.sanitizer = new Sanitizer(this, options); this.nodes = new Nodes(this.options.node_id); - this.observer = new Observer(this, this.options); + this.observer = new Observer(this, options); this.ticker = new Ticker(this); this.ticker.attach(() => this.commit()); try { @@ -102,7 +114,10 @@ export default class App { this.stop(); } else if (data === "restart") { this.stop(); - this.start(true); + this.start({ + forceNew: true, + userID: this._userID, + }); } }; const alertWorker = () => { @@ -244,104 +259,132 @@ export default class App { active(): boolean { return this.isActive; } - private _start(reset: boolean): Promise { - if (!this.isActive) { - if (!this.worker) { - return Promise.reject("No worker found: perhaps, CSP is not set."); - } - this.isActive = true; - let pageNo: number = 0; - const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key); - if (pageNoStr != null) { - pageNo = parseInt(pageNoStr); - pageNo++; - } - sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString()); - const startTimestamp = timestamp(); - - const messageData: WorkerMessageData = { - ingestPoint: this.options.ingestPoint, - pageNo, - startTimestamp, - connAttemptCount: this.options.connAttemptCount, - connAttemptGap: this.options.connAttemptGap, - } - this.worker.postMessage(messageData); // brings delay of 10th ms? - return window.fetch(this.options.ingestPoint + '/v1/web/start', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - token: sessionStorage.getItem(this.options.session_token_key), - userUUID: localStorage.getItem(this.options.local_uuid_key), - projectKey: this.projectKey, - revID: this.revID, - timestamp: startTimestamp, - trackerVersion: this.version, - isSnippet: this.options.__is_snippet, - deviceMemory, - jsHeapSizeLimit, - reset, - }), - }) - .then(r => { - if (r.status === 200) { - return r.json() - } else { // TODO: handle canceling && 403 - return r.text().then(text => { - throw new Error(`Server error: ${r.status}. ${text}`); - }); - } - }) - .then(r => { - const { token, userUUID, sessionID, beaconSizeLimit } = r; - if (typeof token !== 'string' || - typeof userUUID !== 'string' || - (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) { - throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`); - } - sessionStorage.setItem(this.options.session_token_key, token); - localStorage.setItem(this.options.local_uuid_key, userUUID); - if (typeof sessionID === 'string') { - this._sessionID = sessionID; - } - if (!this.worker) { - throw new Error("no worker found after start request (this might not happen)"); - } - this.worker.postMessage({ token, beaconSizeLimit }); - this.startCallbacks.forEach((cb) => cb()); - this.observer.observe(); - this.ticker.start(); - - log("OpenReplay tracking started."); - const onStartInfo = { sessionToken: token, userUUID, sessionID }; - if (typeof this.options.onStart === 'function') { - this.options.onStart(onStartInfo); - } - return onStartInfo; - }) - .catch(e => { - sessionStorage.removeItem(this.options.session_token_key) - this.stop() - warn("OpenReplay was unable to start. ", e) - this._debug("session_start", e); - throw e - }) + resetNextPageSession(flag: boolean) { + if (flag) { + sessionStorage.setItem(this.options.session_reset_key, 't'); + } else { + sessionStorage.removeItem(this.options.session_reset_key); } - return Promise.reject("Player is already active"); + } + private _start(startOpts: StartOptions): Promise { + if (!this.worker) { + return Promise.reject("No worker found: perhaps, CSP is not set."); + } + if (this.isActive) { + return Promise.reject("OpenReplay: trying to call `start()` on the instance that has been started already.") + } + this.isActive = true; + + let pageNo: number = 0; + const pageNoStr = sessionStorage.getItem(this.options.session_pageno_key); + if (pageNoStr != null) { + pageNo = parseInt(pageNoStr); + pageNo++; + } + sessionStorage.setItem(this.options.session_pageno_key, pageNo.toString()); + const startTimestamp = timestamp(); + + const messageData: WorkerMessageData = { + ingestPoint: this.options.ingestPoint, + pageNo, + startTimestamp, + connAttemptCount: this.options.connAttemptCount, + connAttemptGap: this.options.connAttemptGap, + } + this.worker.postMessage(messageData); // brings delay of 10th ms? + + + // let token = sessionStorage.getItem(this.options.session_token_key) + // const tokenIsActive = localStorage.getItem("__or_at_" + token) + // if (tokenIsActive) { + // token = null + // } + + const sReset = sessionStorage.getItem(this.options.session_reset_key); + sessionStorage.removeItem(this.options.session_reset_key); + + this._userID = startOpts.userID || undefined + return window.fetch(this.options.ingestPoint + '/v1/web/start', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + token: sessionStorage.getItem(this.options.session_token_key), + userUUID: localStorage.getItem(this.options.local_uuid_key), + projectKey: this.projectKey, + revID: this.revID, + timestamp: startTimestamp, + trackerVersion: this.version, + isSnippet: this.options.__is_snippet, + deviceMemory, + jsHeapSizeLimit, + reset: startOpts.forceNew || sReset !== null, + userID: this._userID, + }), + }) + .then(r => { + if (r.status === 200) { + return r.json() + } else { // TODO: handle canceling && 403 + return r.text().then(text => { + throw new Error(`Server error: ${r.status}. ${text}`); + }); + } + }) + .then(r => { + const { token, userUUID, sessionID, beaconSizeLimit } = r; + if (typeof token !== 'string' || + typeof userUUID !== 'string' || + (typeof beaconSizeLimit !== 'number' && typeof beaconSizeLimit !== 'undefined')) { + throw new Error(`Incorrect server response: ${ JSON.stringify(r) }`); + } + sessionStorage.setItem(this.options.session_token_key, token); + localStorage.setItem(this.options.local_uuid_key, userUUID); + // localStorage.setItem("__or_at_" + token, "true") + // this.attachEventListener(window, 'beforeunload', ()=>{ + // localStorage.removeItem("__or_at_" + token) + // }, false); + // this.attachEventListener(window, 'pagehide', ()=>{ + // localStorage.removeItem("__or_at_" + token) + // }, false); + if (typeof sessionID === 'string') { + this._sessionID = sessionID; + } + if (!this.worker) { + throw new Error("no worker found after start request (this might not happen)"); + } + this.worker.postMessage({ token, beaconSizeLimit }); + this.startCallbacks.forEach((cb) => cb()); + this.observer.observe(); + this.ticker.start(); + + log("OpenReplay tracking started."); + const onStartInfo = { sessionToken: token, userUUID, sessionID }; + if (typeof this.options.onStart === 'function') { + this.options.onStart(onStartInfo); + } + return onStartInfo; + }) + .catch(e => { + sessionStorage.removeItem(this.options.session_token_key) + this.stop() + warn("OpenReplay was unable to start. ", e) + this._debug("session_start", e); + throw e + }) } - start(reset: boolean = false): Promise { + start(options: StartOptions = { forceNew: false }): Promise { if (!document.hidden) { - return this._start(reset); + return this._start(options); } else { return new Promise((resolve) => { const onVisibilityChange = () => { if (!document.hidden) { document.removeEventListener("visibilitychange", onVisibilityChange); - resolve(this._start(reset)); + resolve(this._start(options)); } } document.addEventListener("visibilitychange", onVisibilityChange); @@ -354,6 +397,7 @@ export default class App { if (this.worker) { this.worker.postMessage("stop"); } + this.sanitizer.clear(); this.observer.disconnect(); this.nodes.clear(); this.ticker.stop(); diff --git a/tracker/tracker/src/main/app/observer.ts b/tracker/tracker/src/main/app/observer.ts deleted file mode 100644 index 3ed5088af..000000000 --- a/tracker/tracker/src/main/app/observer.ts +++ /dev/null @@ -1,484 +0,0 @@ -import { stars, hasOpenreplayAttribute } from "../utils.js"; -import { - CreateDocument, - CreateElementNode, - CreateTextNode, - SetNodeData, - SetCSSDataURLBased, - SetNodeAttribute, - SetNodeAttributeURLBased, - RemoveNodeAttribute, - MoveNode, - RemoveNode, - CreateIFrameDocument, -} from "../../messages/index.js"; -import App from "./index.js"; - -interface Window extends WindowProxy { - HTMLInputElement: typeof HTMLInputElement, - HTMLLinkElement: typeof HTMLLinkElement, - HTMLStyleElement: typeof HTMLStyleElement, - SVGStyleElement: typeof SVGStyleElement, - HTMLIFrameElement: typeof HTMLIFrameElement, - Text: typeof Text, - Element: typeof Element, - //parent: Window, -} - - -type WindowConstructor = - Document | - Element | - Text | - HTMLInputElement | - HTMLLinkElement | - HTMLStyleElement | - HTMLIFrameElement - -// type ConstructorNames = -// 'Element' | -// 'Text' | -// 'HTMLInputElement' | -// 'HTMLLinkElement' | -// 'HTMLStyleElement' | -// 'HTMLIFrameElement' -type Constructor = { new (...args: any[]): T , name: string }; - - -function isSVGElement(node: Element): node is SVGElement { - return node.namespaceURI === 'http://www.w3.org/2000/svg'; -} - -export interface Options { - obscureTextEmails: boolean; - obscureTextNumbers: boolean; - captureIFrames: boolean; -} - -export default class Observer { - private readonly observer: MutationObserver; - private readonly commited: Array; - private readonly recents: Array; - private readonly indexes: Array; - private readonly attributesList: Array | undefined>; - private readonly textSet: Set; - private readonly textMasked: Set; - constructor(private readonly app: App, private readonly options: Options, private readonly context: Window = window) { - this.observer = new MutationObserver( - this.app.safe((mutations) => { - for (const mutation of mutations) { - const target = mutation.target; - const type = mutation.type; - - // Special case - // Document 'childList' might happen in case of iframe. - // TODO: generalize as much as possible - if (this.isInstance(target, Document) - && type === 'childList' - //&& new Array(mutation.addedNodes).some(node => this.isInstance(node, HTMLHtmlElement)) - ) { - const parentFrame = target.defaultView?.frameElement - if (!parentFrame) { continue } - this.bindTree(target.documentElement) - const frameID = this.app.nodes.getID(parentFrame) - const docID = this.app.nodes.getID(target.documentElement) - if (frameID === undefined || docID === undefined) { continue } - this.app.send(CreateIFrameDocument(frameID, docID)); - continue; - } - - if (this.isIgnored(target) || !context.document.contains(target)) { - continue; - } - if (type === 'childList') { - for (let i = 0; i < mutation.removedNodes.length; i++) { - this.bindTree(mutation.removedNodes[i]); - } - for (let i = 0; i < mutation.addedNodes.length; i++) { - this.bindTree(mutation.addedNodes[i]); - } - continue; - } - const id = this.app.nodes.getID(target); - if (id === undefined) { - continue; - } - if (id >= this.recents.length) { - this.recents[id] = undefined; - } - if (type === 'attributes') { - const name = mutation.attributeName; - if (name === null) { - continue; - } - let attr = this.attributesList[id]; - if (attr === undefined) { - this.attributesList[id] = attr = new Set(); - } - attr.add(name); - continue; - } - if (type === 'characterData') { - this.textSet.add(id); - continue; - } - } - this.commitNodes(); - }), - ); - this.commited = []; - this.recents = []; - this.indexes = [0]; - this.attributesList = []; - this.textSet = new Set(); - this.textMasked = new Set(); - } - private clear(): void { - this.commited.length = 0; - this.recents.length = 0; - this.indexes.length = 1; - this.attributesList.length = 0; - this.textSet.clear(); - this.textMasked.clear(); - } - - // TODO: we need a type expert here so we won't have to ignore the lines - private isInstance(node: Node, constr: Constructor): node is T { - let context = this.context; - while(context.parent && context.parent !== context) { - // @ts-ignore - if (node instanceof context[constr.name]) { - return true - } - // @ts-ignore - context = context.parent - } - // @ts-ignore - return node instanceof context[constr.name] - } - - private isIgnored(node: Node): boolean { - if (this.isInstance(node, Text)) { - return false; - } - if (!this.isInstance(node, Element)) { - return true; - } - const tag = node.tagName.toUpperCase(); - if (tag === 'LINK') { - const rel = node.getAttribute('rel'); - const as = node.getAttribute('as'); - return !(rel?.includes('stylesheet') || as === "style" || as === "font"); - } - return ( - tag === 'SCRIPT' || - tag === 'NOSCRIPT' || - tag === 'META' || - tag === 'TITLE' || - tag === 'BASE' - ); - } - - private sendNodeAttribute( - id: number, - node: Element, - name: string, - value: string | null, - ): void { - if (isSVGElement(node)) { - if (name.substr(0, 6) === 'xlink:') { - name = name.substr(6); - } - if (value === null) { - this.app.send(new RemoveNodeAttribute(id, name)); - } else if (name === 'href') { - if (value.length > 1e5) { - value = ''; - } - this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); - } else { - this.app.send(new SetNodeAttribute(id, name, value)); - } - return; - } - if ( - name === 'src' || - name === 'srcset' || - name === 'integrity' || - name === 'crossorigin' || - name === 'autocomplete' || - name.substr(0, 2) === 'on' - ) { - return; - } - if ( - name === 'value' && - this.isInstance(node, HTMLInputElement) && - node.type !== 'button' && - node.type !== 'reset' && - node.type !== 'submit' - ) { - return; - } - if (value === null) { - this.app.send(new RemoveNodeAttribute(id, name)); - return; - } - if (name === 'style' || name === 'href' && this.isInstance(node, HTMLLinkElement)) { - this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); - return; - } - if (name === 'href' || value.length > 1e5) { - value = ''; - } - this.app.send(new SetNodeAttribute(id, name, value)); - } - - /* TODO: abstract sanitation */ - getInnerTextSecure(el: HTMLElement): string { - const id = this.app.nodes.getID(el) - if (!id) { return '' } - return this.checkObscure(id, el.innerText) - - } - - private checkObscure(id: number, data: string): string { - if (this.textMasked.has(id)) { - return data.replace( - /[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g, - '█', - ); - } - if (this.options.obscureTextNumbers) { - data = data.replace(/\d/g, '0'); - } - if (this.options.obscureTextEmails) { - data = data.replace( - /([^\s]+)@([^\s]+)\.([^\s]+)/g, - (...f: Array) => - stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]), - ); - } - return data - } - - private sendNodeData(id: number, parentElement: Element, data: string): void { - if (this.isInstance(parentElement, HTMLStyleElement) || this.isInstance(parentElement, SVGStyleElement)) { - this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref())); - return; - } - data = this.checkObscure(id, data) - this.app.send(new SetNodeData(id, data)); - } - /* end TODO: abstract sanitation */ - - private bindNode(node: Node): void { - const r = this.app.nodes.registerNode(node); - const id = r[0]; - this.recents[id] = r[1] || this.recents[id] || false; - } - - private bindTree(node: Node): void { - if (this.isIgnored(node)) { - return; - } - this.bindNode(node); - const walker = document.createTreeWalker( - node, - NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT, - { - acceptNode: (node) => - this.isIgnored(node) || this.app.nodes.getID(node) !== undefined - ? NodeFilter.FILTER_REJECT - : NodeFilter.FILTER_ACCEPT, - }, - // @ts-ignore - false, - ); - while (walker.nextNode()) { - this.bindNode(walker.currentNode); - } - } - - private unbindNode(node: Node): void { - const id = this.app.nodes.unregisterNode(node); - if (id !== undefined && this.recents[id] === false) { - this.app.send(new RemoveNode(id)); - } - } - - private _commitNode(id: number, node: Node): boolean { - const parent = node.parentNode; - let parentID: number | undefined; - if (this.isInstance(node, HTMLHtmlElement)) { - this.indexes[id] = 0 - } else { - if (parent === null) { - this.unbindNode(node); - return false; - } - parentID = this.app.nodes.getID(parent); - if (parentID === undefined) { - this.unbindNode(node); - return false; - } - if (!this.commitNode(parentID)) { - this.unbindNode(node); - return false; - } - if ( - this.textMasked.has(parentID) || - (this.isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked')) - ) { - this.textMasked.add(id); - } - let sibling = node.previousSibling; - while (sibling !== null) { - const siblingID = this.app.nodes.getID(sibling); - if (siblingID !== undefined) { - this.commitNode(siblingID); - this.indexes[id] = this.indexes[siblingID] + 1; - break; - } - sibling = sibling.previousSibling; - } - if (sibling === null) { - this.indexes[id] = 0; - } - } - const isNew = this.recents[id]; - const index = this.indexes[id]; - if (index === undefined) { - throw 'commitNode: missing node index'; - } - if (isNew === true) { - if (this.isInstance(node, Element)) { - if (parentID !== undefined) { - this.app.send(new - CreateElementNode( - id, - parentID, - index, - node.tagName, - isSVGElement(node), - ), - ); - } - for (let i = 0; i < node.attributes.length; i++) { - const attr = node.attributes[i]; - this.sendNodeAttribute(id, node, attr.nodeName, attr.value); - } - - if (this.isInstance(node, HTMLIFrameElement) && - (this.options.captureIFrames || node.getAttribute("data-openreplay-capture"))) { - this.handleIframe(node); - } - } else if (this.isInstance(node, Text)) { - // for text node id != 0, hence parentID !== undefined and parent is Element - this.app.send(new CreateTextNode(id, parentID as number, index)); - this.sendNodeData(id, parent as Element, node.data); - } - return true; - } - if (isNew === false && parentID !== undefined) { - this.app.send(new MoveNode(id, parentID, index)); - } - const attr = this.attributesList[id]; - if (attr !== undefined) { - if (!this.isInstance(node, Element)) { - throw 'commitNode: node is not an element'; - } - for (const name of attr) { - this.sendNodeAttribute(id, node, name, node.getAttribute(name)); - } - } - if (this.textSet.has(id)) { - if (!this.isInstance(node, Text)) { - throw 'commitNode: node is not a text'; - } - // for text node id != 0, hence parent is Element - this.sendNodeData(id, parent as Element, node.data); - } - return true; - } - private commitNode(id: number): boolean { - const node = this.app.nodes.getNode(id); - if (node === undefined) { - return false; - } - const cmt = this.commited[id]; - if (cmt !== undefined) { - return cmt; - } - return (this.commited[id] = this._commitNode(id, node)); - } - private commitNodes(): void { - let node; - for (let id = 0; id < this.recents.length; id++) { - this.commitNode(id); - if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) { - this.app.nodes.callNodeCallbacks(node); - } - } - this.clear(); - } - - private iframeObservers: Observer[] = []; - private handleIframe(iframe: HTMLIFrameElement): void { - let context: Window | null = null - const handle = this.app.safe(() => { - const id = this.app.nodes.getID(iframe) - if (id === undefined) { return } - if (iframe.contentWindow === context) { return } - context = iframe.contentWindow as Window | null; - if (!context) { return } - const observer = new Observer(this.app, this.options, context) - this.iframeObservers.push(observer) - observer.observeIframe(id, context) - }) - this.app.attachEventListener(iframe, "load", handle) - handle() - } - - // TODO: abstract common functionality, separate FrameObserver - private observeIframe(id: number, context: Window) { - const doc = context.document; - this.observer.observe(doc, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: false, - characterDataOldValue: false, - }); - this.bindTree(doc.documentElement); - const docID = this.app.nodes.getID(doc.documentElement); - if (docID === undefined) { - console.log("Wrong") - return; - } - this.app.send(CreateIFrameDocument(id,docID)); - this.commitNodes(); - } - - observe(): void { - this.observer.observe(this.context.document, { - childList: true, - attributes: true, - characterData: true, - subtree: true, - attributeOldValue: false, - characterDataOldValue: false, - }); - this.app.send(new CreateDocument()); - this.bindTree(this.context.document.documentElement); - this.commitNodes(); - } - - disconnect(): void { - this.iframeObservers.forEach(o => o.disconnect()); - this.iframeObservers = []; - this.observer.disconnect(); - this.clear(); - } -} diff --git a/tracker/tracker/src/main/app/observer/iframe_observer.ts b/tracker/tracker/src/main/app/observer/iframe_observer.ts new file mode 100644 index 000000000..be0a7182c --- /dev/null +++ b/tracker/tracker/src/main/app/observer/iframe_observer.ts @@ -0,0 +1,19 @@ +import Observer from "./observer.js"; +import { CreateIFrameDocument } from "../../../messages/index.js"; + +export default class IFrameObserver extends Observer { + observe(iframe: HTMLIFrameElement) { + const doc = iframe.contentDocument; + const hostID = this.app.nodes.getID(iframe); + if (!doc || hostID === undefined) { return } //log TODO common app.logger + // Have to observe document, because the inner might be changed + this.observeRoot(doc, (docID) => { + if (docID === undefined) { + console.log("OpenReplay: Iframe document not bound") + return; + } + this.app.send(CreateIFrameDocument(hostID, docID)); + }); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/observer/observer.ts b/tracker/tracker/src/main/app/observer/observer.ts new file mode 100644 index 000000000..0f4ff2994 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/observer.ts @@ -0,0 +1,353 @@ +import { hasOpenreplayAttribute } from "../../utils.js"; +import { + RemoveNodeAttribute, + SetNodeAttribute, + SetNodeAttributeURLBased, + SetCSSDataURLBased, + SetNodeData, + CreateTextNode, + CreateElementNode, + MoveNode, + RemoveNode, +} from "../../../messages/index.js"; +import App from "../index.js"; +import { isInstance, inDocument } from "../context.js"; + + +function isSVGElement(node: Element): node is SVGElement { + return node.namespaceURI === 'http://www.w3.org/2000/svg'; +} + +function isIgnored(node: Node): boolean { + if (isInstance(node, Text)) { + return false; + } + if (!isInstance(node, Element)) { + return true; + } + const tag = node.tagName.toUpperCase(); + if (tag === 'LINK') { + const rel = node.getAttribute('rel'); + const as = node.getAttribute('as'); + return !(rel?.includes('stylesheet') || as === "style" || as === "font"); + } + return ( + tag === 'SCRIPT' || + tag === 'NOSCRIPT' || + tag === 'META' || + tag === 'TITLE' || + tag === 'BASE' + ); +} + +function isRootNode(node: Node): boolean { + return isInstance(node, Document) || isInstance(node, ShadowRoot); +} + +function isObservable(node: Node): boolean { + if (isRootNode(node)) { + return true; + } + return !isIgnored(node); +} + +export default abstract class Observer { + private readonly observer: MutationObserver; + private readonly commited: Array = []; + private readonly recents: Array = []; + private readonly myNodes: Array = []; + private readonly indexes: Array = []; + private readonly attributesList: Array | undefined> = []; + private readonly textSet: Set = new Set(); + private readonly inUpperContext: boolean; + constructor(protected readonly app: App, protected readonly context: Window = window) { + this.inUpperContext = context.parent === context //TODO: get rid of context here + this.observer = new MutationObserver( + this.app.safe((mutations) => { + for (const mutation of mutations) { + const target = mutation.target; + const type = mutation.type; + + if (!isObservable(target) || !inDocument(target)) { + continue; + } + if (type === 'childList') { + for (let i = 0; i < mutation.removedNodes.length; i++) { + this.bindTree(mutation.removedNodes[i]); + } + for (let i = 0; i < mutation.addedNodes.length; i++) { + this.bindTree(mutation.addedNodes[i]); + } + continue; + } + const id = this.app.nodes.getID(target); + if (id === undefined) { + continue; + } + if (id >= this.recents.length) { // TODO: something more convinient + this.recents[id] = undefined; + } + if (type === 'attributes') { + const name = mutation.attributeName; + if (name === null) { + continue; + } + let attr = this.attributesList[id]; + if (attr === undefined) { + this.attributesList[id] = attr = new Set(); + } + attr.add(name); + continue; + } + if (type === 'characterData') { + this.textSet.add(id); + continue; + } + } + this.commitNodes(); + }), + ); + } + private clear(): void { + this.commited.length = 0; + this.recents.length = 0; + this.indexes.length = 1; + this.attributesList.length = 0; + this.textSet.clear(); + } + + private sendNodeAttribute( + id: number, + node: Element, + name: string, + value: string | null, + ): void { + if (isSVGElement(node)) { + if (name.substr(0, 6) === 'xlink:') { + name = name.substr(6); + } + if (value === null) { + this.app.send(new RemoveNodeAttribute(id, name)); + } else if (name === 'href') { + if (value.length > 1e5) { + value = ''; + } + this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); + } else { + this.app.send(new SetNodeAttribute(id, name, value)); + } + return; + } + if ( + name === 'src' || + name === 'srcset' || + name === 'integrity' || + name === 'crossorigin' || + name === 'autocomplete' || + name.substr(0, 2) === 'on' + ) { + return; + } + if ( + name === 'value' && + isInstance(node, HTMLInputElement) && + node.type !== 'button' && + node.type !== 'reset' && + node.type !== 'submit' + ) { + return; + } + if (value === null) { + this.app.send(new RemoveNodeAttribute(id, name)); + return; + } + if (name === 'style' || name === 'href' && isInstance(node, HTMLLinkElement)) { + this.app.send(new SetNodeAttributeURLBased(id, name, value, this.app.getBaseHref())); + return; + } + if (name === 'href' || value.length > 1e5) { + value = ''; + } + this.app.send(new SetNodeAttribute(id, name, value)); + } + + private sendNodeData(id: number, parentElement: Element, data: string): void { + if (isInstance(parentElement, HTMLStyleElement) || isInstance(parentElement, SVGStyleElement)) { + this.app.send(new SetCSSDataURLBased(id, data, this.app.getBaseHref())); + return; + } + data = this.app.sanitizer.sanitize(id, data) + this.app.send(new SetNodeData(id, data)); + } + + private bindNode(node: Node): void { + const r = this.app.nodes.registerNode(node); + const id = r[0]; + this.recents[id] = r[1] || this.recents[id] || false; + + this.myNodes[id] = true; + } + + private bindTree(node: Node): void { + if (!isObservable(node)) { + return + } + this.bindNode(node); + const walker = document.createTreeWalker( + node, + NodeFilter.SHOW_ELEMENT + NodeFilter.SHOW_TEXT, + { + acceptNode: (node) => + isIgnored(node) || this.app.nodes.getID(node) !== undefined + ? NodeFilter.FILTER_REJECT + : NodeFilter.FILTER_ACCEPT, + }, + // @ts-ignore + false, + ); + while (walker.nextNode()) { + this.bindNode(walker.currentNode); + } + } + + private unbindNode(node: Node): void { + const id = this.app.nodes.unregisterNode(node); + if (id !== undefined && this.recents[id] === false) { + this.app.send(new RemoveNode(id)); + } + } + + private _commitNode(id: number, node: Node): boolean { + if (isRootNode(node)) { + return true; + } + const parent = node.parentNode; + let parentID: number | undefined; + // Disable parent check for the upper context HTMLHtmlElement, because it is root there... (before) + // TODO: get rid of "special" cases (there is an issue with CreateDocument altered behaviour though) + // TODO: Clean the logic (though now it workd fine) + if (!isInstance(node, HTMLHtmlElement) || !this.inUpperContext) { + if (parent === null) { + this.unbindNode(node); + return false; + } + parentID = this.app.nodes.getID(parent); + if (parentID === undefined) { + this.unbindNode(node); + return false; + } + if (!this.commitNode(parentID)) { + this.unbindNode(node); + return false; + } + this.app.sanitizer.handleNode(id, parentID, node); + } + let sibling = node.previousSibling; + while (sibling !== null) { + const siblingID = this.app.nodes.getID(sibling); + if (siblingID !== undefined) { + this.commitNode(siblingID); + this.indexes[id] = this.indexes[siblingID] + 1; + break; + } + sibling = sibling.previousSibling; + } + if (sibling === null) { + this.indexes[id] = 0; // + } + const isNew = this.recents[id]; + const index = this.indexes[id]; + if (index === undefined) { + throw 'commitNode: missing node index'; + } + if (isNew === true) { + if (isInstance(node, Element)) { + if (parentID !== undefined) { + this.app.send(new + CreateElementNode( + id, + parentID, + index, + node.tagName, + isSVGElement(node), + ), + ); + } + for (let i = 0; i < node.attributes.length; i++) { + const attr = node.attributes[i]; + this.sendNodeAttribute(id, node, attr.nodeName, attr.value); + } + } else if (isInstance(node, Text)) { + // for text node id != 0, hence parentID !== undefined and parent is Element + this.app.send(new CreateTextNode(id, parentID as number, index)); + this.sendNodeData(id, parent as Element, node.data); + } + return true; + } + if (isNew === false && parentID !== undefined) { + this.app.send(new MoveNode(id, parentID, index)); + } + const attr = this.attributesList[id]; + if (attr !== undefined) { + if (!isInstance(node, Element)) { + throw 'commitNode: node is not an element'; + } + for (const name of attr) { + this.sendNodeAttribute(id, node, name, node.getAttribute(name)); + } + } + if (this.textSet.has(id)) { + if (!isInstance(node, Text)) { + throw 'commitNode: node is not a text'; + } + // for text node id != 0, hence parent is Element + this.sendNodeData(id, parent as Element, node.data); + } + return true; + } + private commitNode(id: number): boolean { + const node = this.app.nodes.getNode(id); + if (node === undefined) { + return false; + } + const cmt = this.commited[id]; + if (cmt !== undefined) { + return cmt; + } + return (this.commited[id] = this._commitNode(id, node)); + } + private commitNodes(): void { + let node; + for (let id = 0; id < this.recents.length; id++) { + // TODO: make things/logic nice here. + // commit required in any case if recents[id] true or false (in case of unbinding) or undefined (in case of attr change). + if (!this.myNodes[id]) { continue } + this.commitNode(id); + if (this.recents[id] === true && (node = this.app.nodes.getNode(id))) { + this.app.nodes.callNodeCallbacks(node); + } + } + this.clear(); + } + + // ISSSUE + protected observeRoot(node: Node, beforeCommit: (id?: number) => unknown, nodeToBind: Node = node) { + this.observer.observe(node, { + childList: true, + attributes: true, + characterData: true, + subtree: true, + attributeOldValue: false, + characterDataOldValue: false, + }); + this.bindTree(nodeToBind); + beforeCommit(this.app.nodes.getID(node)) + this.commitNodes(); + } + + disconnect(): void { + this.observer.disconnect(); + this.clear(); + this.myNodes.length = 0; + } +} diff --git a/tracker/tracker/src/main/app/observer/shadow_root_observer.ts b/tracker/tracker/src/main/app/observer/shadow_root_observer.ts new file mode 100644 index 000000000..244348ea1 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/shadow_root_observer.ts @@ -0,0 +1,18 @@ +import Observer from "./observer.js"; +import { CreateIFrameDocument } from "../../../messages/index.js"; + +export default class ShadowRootObserver extends Observer { + observe(el: Element) { + const shRoot = el.shadowRoot; + const hostID = this.app.nodes.getID(el); + if (!shRoot || hostID === undefined) { return } // log + this.observeRoot(shRoot, (rootID) => { + if (rootID === undefined) { + console.log("OpenReplay: Shadow Root was not bound") + return; + } + this.app.send(CreateIFrameDocument(hostID,rootID)); + }); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/observer/top_observer.ts b/tracker/tracker/src/main/app/observer/top_observer.ts new file mode 100644 index 000000000..b35f5d901 --- /dev/null +++ b/tracker/tracker/src/main/app/observer/top_observer.ts @@ -0,0 +1,98 @@ +import Observer from "./observer.js"; +import { isInstance } from "../context.js"; +import type { Window } from "../context.js"; +import IFrameObserver from "./iframe_observer.js"; +import ShadowRootObserver from "./shadow_root_observer.js"; + +import { CreateDocument } from "../../../messages/index.js"; +import App from "../index.js"; +import { IN_BROWSER } from '../../utils.js' + +export interface Options { + captureIFrames: boolean +} + +const attachShadowNativeFn = IN_BROWSER ? Element.prototype.attachShadow : ()=>new ShadowRoot(); + +export default class TopObserver extends Observer { + private readonly options: Options; + constructor(app: App, options: Partial) { + super(app); + this.options = Object.assign({ + captureIFrames: false + }, options); + + // IFrames + this.app.nodes.attachNodeCallback(node => { + if (isInstance(node, HTMLIFrameElement) && + (this.options.captureIFrames || node.getAttribute("data-openreplay-capture")) + ) { + this.handleIframe(node) + } + }) + + // ShadowDOM + this.app.nodes.attachNodeCallback(node => { + if (isInstance(node, Element) && node.shadowRoot !== null) { + this.handleShadowRoot(node.shadowRoot) + } + }) + } + + + private iframeObservers: IFrameObserver[] = []; + private handleIframe(iframe: HTMLIFrameElement): void { + let context: Window | null = null + const handle = this.app.safe(() => { + const id = this.app.nodes.getID(iframe) + if (id === undefined) { return } //log + if (iframe.contentWindow === context) { return } //Does this happen frequently? + context = iframe.contentWindow as Window | null; + if (!context) { return } + const observer = new IFrameObserver(this.app, context) + + this.iframeObservers.push(observer) + observer.observe(iframe) + }) + this.app.attachEventListener(iframe, "load", handle) + handle() + } + + private shadowRootObservers: ShadowRootObserver[] = [] + private handleShadowRoot(shRoot: ShadowRoot) { + const observer = new ShadowRootObserver(this.app, this.context) + + this.shadowRootObservers.push(observer) + observer.observe(shRoot.host) + } + + observe(): void { + // Protection from several subsequent calls? + const observer = this; + Element.prototype.attachShadow = function() { + const shadow = attachShadowNativeFn.apply(this, arguments) + observer.handleShadowRoot(shadow) + return shadow + } + + // Can observe documentElement () here, because it is not supposed to be changing. + // However, it is possible in some exotic cases and may cause an ignorance of the newly created + // In this case context.document have to be observed, but this will cause + // the change in the re-player behaviour caused by CreateDocument message: + // the 0-node ("fRoot") will become #document rather than documentElement as it is now. + // Alternatively - observe(#document) then bindNode(documentElement) + this.observeRoot(this.context.document, () => { + this.app.send(new CreateDocument()) + }, this.context.document.documentElement); + } + + disconnect() { + Element.prototype.attachShadow = attachShadowNativeFn + this.iframeObservers.forEach(o => o.disconnect()) + this.iframeObservers = [] + this.shadowRootObservers.forEach(o => o.disconnect()) + this.shadowRootObservers = [] + super.disconnect() + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/app/sanitizer.ts b/tracker/tracker/src/main/app/sanitizer.ts new file mode 100644 index 000000000..d085b5739 --- /dev/null +++ b/tracker/tracker/src/main/app/sanitizer.ts @@ -0,0 +1,66 @@ +import { stars, hasOpenreplayAttribute } from "../utils.js"; +import App from "./index.js"; +import { isInstance } from "./context.js"; + +export interface Options { + obscureTextEmails: boolean; + obscureTextNumbers: boolean; +} + +export default class Sanitizer { + private readonly masked: Set = new Set(); + private readonly options: Options; + + constructor(private readonly app: App, options: Partial) { + this.options = Object.assign({ + obscureTextEmails: true, + obscureTextNumbers: false, + }, options); + } + + handleNode(id: number, parentID: number, node: Node) { + if ( + this.masked.has(parentID) || + (isInstance(node, Element) && hasOpenreplayAttribute(node, 'masked')) + ) { + this.masked.add(id); + } + } + + sanitize(id: number, data: string): string { + if (this.masked.has(id)) { + // TODO: is it the best place to put trim() ? Might trimmed spaces be considered in layout in certain cases? + return data.trim().replace( + /[^\f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]/g, + '█', + ); + } + if (this.options.obscureTextNumbers) { + data = data.replace(/\d/g, '0'); + } + if (this.options.obscureTextEmails) { + data = data.replace( + /([^\s]+)@([^\s]+)\.([^\s]+)/g, + (...f: Array) => + stars(f[1]) + '@' + stars(f[2]) + '.' + stars(f[3]), + ); + } + return data + } + + isMasked(id: number): boolean { + return this.masked.has(id); + } + + getInnerTextSecure(el: HTMLElement): string { + const id = this.app.nodes.getID(el) + if (!id) { return '' } + return this.sanitize(id, el.innerText) + + } + + clear(): void { + this.masked.clear(); + } + +} \ No newline at end of file diff --git a/tracker/tracker/src/main/index.ts b/tracker/tracker/src/main/index.ts index 6af325e57..75d195e50 100644 --- a/tracker/tracker/src/main/index.ts +++ b/tracker/tracker/src/main/index.ts @@ -19,14 +19,15 @@ import Longtasks from "./modules/longtasks.js"; import CSSRules from "./modules/cssrules.js"; import { IN_BROWSER, deprecationWarn, DOCS_HOST } from "./utils.js"; -import { Options as AppOptions } from "./app/index.js"; -import { Options as ConsoleOptions } from "./modules/console.js"; -import { Options as ExceptionOptions } from "./modules/exception.js"; -import { Options as InputOptions } from "./modules/input.js"; -import { Options as PerformanceOptions } from "./modules/performance.js"; -import { Options as TimingOptions } from "./modules/timing.js"; - -export type { OnStartInfo } from './app/index.js'; +import type { Options as AppOptions } from "./app/index.js"; +import type { Options as ConsoleOptions } from "./modules/console.js"; +import type { Options as ExceptionOptions } from "./modules/exception.js"; +import type { Options as InputOptions } from "./modules/input.js"; +import type { Options as PerformanceOptions } from "./modules/performance.js"; +import type { Options as TimingOptions } from "./modules/timing.js"; +import type { StartOptions } from './app/index.js' +//TODO: unique options init +import type { OnStartInfo } from './app/index.js'; export type Options = Partial< AppOptions & ConsoleOptions & ExceptionOptions & InputOptions & PerformanceOptions & TimingOptions @@ -35,6 +36,7 @@ export type Options = Partial< projectKey: string; sessionToken?: string; respectDoNotTrack?: boolean; + autoResetOnWindowOpen?: boolean; // dev only __DISABLE_SECURE_MODE?: boolean; }; @@ -84,7 +86,7 @@ export default class API { (navigator.doNotTrack == '1' // @ts-ignore || window.doNotTrack == '1'); - this.app = doNotTrack || + const app = this.app = doNotTrack || !('Map' in window) || !('Set' in window) || !('MutationObserver' in window) || @@ -95,20 +97,35 @@ export default class API { !('Worker' in window) ? null : new App(options.projectKey, options.sessionToken, options); - if (this.app !== null) { - Viewport(this.app); - CSSRules(this.app); - Connection(this.app); - Console(this.app, options); - Exception(this.app, options); - Img(this.app); - Input(this.app, options); - Mouse(this.app); - Timing(this.app, options); - Performance(this.app, options); - Scroll(this.app); - Longtasks(this.app); + if (app !== null) { + Viewport(app); + CSSRules(app); + Connection(app); + Console(app, options); + Exception(app, options); + Img(app); + Input(app, options); + Mouse(app); + Timing(app, options); + Performance(app, options); + Scroll(app); + Longtasks(app); (window as any).__OPENREPLAY__ = this; + + if (options.autoResetOnWindowOpen) { + const wOpen = window.open; + app.attachStartCallback(() => { + // @ts-ignore ? + window.open = function(...args) { + app.resetNextPageSession(true) + wOpen.call(window, ...args) + app.resetNextPageSession(false) + } + }) + app.attachStopCallback(() => { + window.open = wOpen; + }) + } } else { console.log("OpenReplay: browser doesn't support API required for tracking or doNotTrack is set to 1.") const req = new XMLHttpRequest(); @@ -140,7 +157,7 @@ export default class API { return this.isActive(); } - start() /*: Promise*/ { + start(startOpts?: StartOptions) : Promise { if (!IN_BROWSER) { console.error(`OpenReplay: you are trying to start Tracker on a node.js environment. If you want to use OpenReplay with SSR, please, use componentDidMount or useEffect API for placing the \`tracker.start()\` line. Check documentation on ${DOCS_HOST}${DOCS_SETUP}`) return Promise.reject("Trying to start not in browser."); @@ -148,7 +165,7 @@ export default class API { if (this.app === null) { return Promise.reject("Browser doesn't support required api, or doNotTrack is active."); } - return this.app.start(); + return this.app.start(startOpts); } stop(): void { if (this.app === null) { diff --git a/tracker/tracker/src/main/modules/exception.ts b/tracker/tracker/src/main/modules/exception.ts index 45fe37465..848df03be 100644 --- a/tracker/tracker/src/main/modules/exception.ts +++ b/tracker/tracker/src/main/modules/exception.ts @@ -50,7 +50,13 @@ export function getExceptionMessageFromEvent(e: ErrorEvent | PromiseRejectionEve if (e.reason instanceof Error) { return getExceptionMessage(e.reason, []) } else { - return new JSException('Unhandled Promise Rejection', String(e.reason), '[]'); + let message: string; + try { + message = JSON.stringify(e.reason) + } catch(_) { + message = String(e.reason) + } + return new JSException('Unhandled Promise Rejection', message, '[]'); } } return null; diff --git a/tracker/tracker/src/main/modules/img.ts b/tracker/tracker/src/main/modules/img.ts index 61e793b89..8c0f911a8 100644 --- a/tracker/tracker/src/main/modules/img.ts +++ b/tracker/tracker/src/main/modules/img.ts @@ -1,8 +1,21 @@ import { timestamp, isURL } from "../utils.js"; import App from "../app/index.js"; -import { ResourceTiming, SetNodeAttributeURLBased } from "../../messages/index.js"; +import { ResourceTiming, SetNodeAttributeURLBased, SetNodeAttribute } from "../../messages/index.js"; + +const PLACEHOLDER_SRC = "https://static.openreplay.com/tracker/placeholder.jpeg"; export default function (app: App): void { + function sendPlaceholder(id: number, node: HTMLImageElement): void { + app.send(new SetNodeAttribute(id, "src", PLACEHOLDER_SRC)) + const { width, height } = node.getBoundingClientRect(); + if (!node.hasAttribute("width")){ + app.send(new SetNodeAttribute(id, "width", String(width))) + } + if (!node.hasAttribute("height")){ + app.send(new SetNodeAttribute(id, "height", String(height))) + } + } + const sendImgSrc = app.safe(function (this: HTMLImageElement): void { const id = app.nodes.getID(this); if (id === undefined) { @@ -16,7 +29,9 @@ export default function (app: App): void { if (src != null && isURL(src)) { // TODO: How about relative urls ? Src type is null sometimes. app.send(new ResourceTiming(timestamp(), 0, 0, 0, 0, 0, src, 'img')); } - } else if (src.length < 1e5) { + } else if (src.length >= 1e5 || app.sanitizer.isMasked(id)) { + sendPlaceholder(id, this) + } else { app.send(new SetNodeAttributeURLBased(id, 'src', src, app.getBaseHref())); } }); diff --git a/tracker/tracker/src/main/modules/input.ts b/tracker/tracker/src/main/modules/input.ts index 746c26f8f..ad8cda673 100644 --- a/tracker/tracker/src/main/modules/input.ts +++ b/tracker/tracker/src/main/modules/input.ts @@ -2,7 +2,12 @@ import { normSpaces, IN_BROWSER, getLabelAttribute, hasOpenreplayAttribute } fro import App from "../app/index.js"; import { SetInputTarget, SetInputValue, SetInputChecked } from "../../messages/index.js"; -function isInput(node: any): node is HTMLInputElement { +// TODO: take into consideration "contenteditable" attribute +type TextEditableElement = HTMLInputElement | HTMLTextAreaElement +function isTextEditable(node: any): node is TextEditableElement { + if (node instanceof HTMLTextAreaElement) { + return true; + } if (!(node instanceof HTMLInputElement)) { return false; } @@ -16,6 +21,7 @@ function isInput(node: any): node is HTMLInputElement { type === 'range' ); } + function isCheckable(node: any): node is HTMLInputElement { if (!(node instanceof HTMLInputElement)) { return false; @@ -25,7 +31,7 @@ function isCheckable(node: any): node is HTMLInputElement { } const labelElementFor: ( - node: HTMLInputElement, + node: TextEditableElement, ) => HTMLLabelElement | undefined = IN_BROWSER && 'labels' in HTMLInputElement.prototype ? (node): HTMLLabelElement | undefined => { @@ -56,7 +62,7 @@ const labelElementFor: ( } }; -export function getInputLabel(node: HTMLInputElement): string { +export function getInputLabel(node: TextEditableElement): string { let label = getLabelAttribute(node); if (label === null) { const labelElement = labelElementFor(node); @@ -89,13 +95,13 @@ export default function (app: App, opts: Partial): void { }, opts, ); - function sendInputTarget(id: number, node: HTMLInputElement): void { + function sendInputTarget(id: number, node: TextEditableElement): void { const label = getInputLabel(node); if (label !== '') { app.send(new SetInputTarget(id, label)); } } - function sendInputValue(id: number, node: HTMLInputElement): void { + function sendInputValue(id: number, node: TextEditableElement): void { let value = node.value; let inputMode: InputMode = options.defaultInputMode; if (node.type === 'password' || hasOpenreplayAttribute(node, 'hidden')) { @@ -136,7 +142,7 @@ export default function (app: App, opts: Partial): void { app.ticker.attach((): void => { inputValues.forEach((value, id) => { const node = app.nodes.getNode(id); - if (!isInput(node)) { + if (!isTextEditable(node)) { inputValues.delete(id); return; } @@ -169,7 +175,7 @@ export default function (app: App, opts: Partial): void { if (id === undefined) { return; } - if (isInput(node)) { + if (isTextEditable(node)) { inputValues.set(id, node.value); sendInputValue(id, node); return; diff --git a/tracker/tracker/src/main/modules/mouse.ts b/tracker/tracker/src/main/modules/mouse.ts index 3ec70e844..0089fa37f 100644 --- a/tracker/tracker/src/main/modules/mouse.ts +++ b/tracker/tracker/src/main/modules/mouse.ts @@ -92,7 +92,7 @@ export default function (app: App): void { (target as HTMLElement).onclick != null || target.getAttribute('role') === 'button' ) { - const label: string = app.observer.getInnerTextSecure(target as HTMLElement); + const label: string = app.sanitizer.getInnerTextSecure(target as HTMLElement); return normSpaces(label).slice(0, 100); } return ''; diff --git a/tracker/tracker/src/webworker/index.ts b/tracker/tracker/src/webworker/index.ts index 723008c52..cf0d1586a 100644 --- a/tracker/tracker/src/webworker/index.ts +++ b/tracker/tracker/src/webworker/index.ts @@ -47,6 +47,7 @@ function sendBatch(batch: Uint8Array):void { return; // happens simultaneously with onerror TODO: clear codeflow } if (this.status >= 400) { // TODO: test workflow. After 400+ it calls /start for some reason + busy = false; reset(); sendQueue.length = 0; if (this.status === 401) { // Unauthorised (Token expired) From 12b414793521a5dfe5f83c8316ff802ac6e5f89a Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 18:57:06 +0100 Subject: [PATCH 185/206] feat (backend): insert userID on SessionStart --- backend/pkg/db/postgres/messages_common.go | 7 +++++-- backend/pkg/messages/messages.go | 4 +++- backend/pkg/messages/read_message.go | 1 + backend/services/http/handlers_web.go | 2 ++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/pkg/db/postgres/messages_common.go b/backend/pkg/db/postgres/messages_common.go index a6f624651..df539e05c 100644 --- a/backend/pkg/db/postgres/messages_common.go +++ b/backend/pkg/db/postgres/messages_common.go @@ -47,7 +47,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { rev_id, tracker_version, issue_score, platform, - user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size + user_agent, user_browser, user_browser_version, user_device_memory_size, user_device_heap_size, + user_id ) VALUES ( $1, $2, $3, $4, $5, $6, $7, @@ -55,7 +56,8 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { NULLIF($10, ''), $11, $12, $13, - NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint) + NULLIF($14, ''), NULLIF($15, ''), NULLIF($16, ''), NULLIF($17, 0), NULLIF($18, 0::bigint), + NULLIF($19, '') )`, sessionID, s.ProjectID, s.Timestamp, s.UserUUID, s.UserDevice, s.UserDeviceType, s.UserCountry, @@ -64,6 +66,7 @@ func (conn *Conn) InsertSessionStart(sessionID uint64, s *types.Session) error { s.TrackerVersion, s.Timestamp/1000, s.Platform, s.UserAgent, s.UserBrowser, s.UserBrowserVersion, s.UserDeviceMemorySize, s.UserDeviceHeapSize, + s.UserID, ); err != nil { return err; } diff --git a/backend/pkg/messages/messages.go b/backend/pkg/messages/messages.go index 3d8bae7f6..cdff71e1d 100644 --- a/backend/pkg/messages/messages.go +++ b/backend/pkg/messages/messages.go @@ -63,9 +63,10 @@ UserDeviceType string UserDeviceMemorySize uint64 UserDeviceHeapSize uint64 UserCountry string +UserID string } func (msg *SessionStart) Encode() []byte{ - buf := make([]byte, 151 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)) + buf := make([]byte, 161 + len(msg.TrackerVersion)+ len(msg.RevID)+ len(msg.UserUUID)+ len(msg.UserAgent)+ len(msg.UserOS)+ len(msg.UserOSVersion)+ len(msg.UserBrowser)+ len(msg.UserBrowserVersion)+ len(msg.UserDevice)+ len(msg.UserDeviceType)+ len(msg.UserCountry)+ len(msg.UserID)) buf[0] = 1 p := 1 p = WriteUint(msg.Timestamp, buf, p) @@ -83,6 +84,7 @@ p = WriteString(msg.UserDeviceType, buf, p) p = WriteUint(msg.UserDeviceMemorySize, buf, p) p = WriteUint(msg.UserDeviceHeapSize, buf, p) p = WriteString(msg.UserCountry, buf, p) +p = WriteString(msg.UserID, buf, p) return buf[:p] } diff --git a/backend/pkg/messages/read_message.go b/backend/pkg/messages/read_message.go index d0148bbc6..c226df728 100644 --- a/backend/pkg/messages/read_message.go +++ b/backend/pkg/messages/read_message.go @@ -42,6 +42,7 @@ if msg.UserDeviceType, err = ReadString(reader); err != nil { return nil, err } if msg.UserDeviceMemorySize, err = ReadUint(reader); err != nil { return nil, err } if msg.UserDeviceHeapSize, err = ReadUint(reader); err != nil { return nil, err } if msg.UserCountry, err = ReadString(reader); err != nil { return nil, err } +if msg.UserID, err = ReadString(reader); err != nil { return nil, err } return msg, nil case 2: diff --git a/backend/services/http/handlers_web.go b/backend/services/http/handlers_web.go index 5e144f1cc..09d2511d8 100644 --- a/backend/services/http/handlers_web.go +++ b/backend/services/http/handlers_web.go @@ -27,6 +27,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { JsHeapSizeLimit uint64 `json:"jsHeapSizeLimit"` ProjectKey *string `json:"projectKey"` Reset bool `json:"reset"` + UserID string `json:"userID"` } type response struct { Timestamp int64 `json:"timestamp"` @@ -101,6 +102,7 @@ func startSessionHandlerWeb(w http.ResponseWriter, r *http.Request) { UserCountry: country, UserDeviceMemorySize: req.DeviceMemory, UserDeviceHeapSize: req.JsHeapSizeLimit, + UserID: req.UserID, })) } From 41cd2aba7b37efab3c4b1d86c0eb83f723257374 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 19:31:13 +0100 Subject: [PATCH 186/206] feat-fix (backend): insert userID on SessionStart (2) --- backend/pkg/db/cache/messages_web.go | 1 + backend/pkg/db/types/session.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/pkg/db/cache/messages_web.go b/backend/pkg/db/cache/messages_web.go index 3afae8592..b259e49da 100644 --- a/backend/pkg/db/cache/messages_web.go +++ b/backend/pkg/db/cache/messages_web.go @@ -30,6 +30,7 @@ func (c *PGCache) InsertWebSessionStart(sessionID uint64, s *SessionStart) error UserDeviceType: s.UserDeviceType, UserDeviceMemorySize: s.UserDeviceMemorySize, UserDeviceHeapSize: s.UserDeviceHeapSize, + UserID: s.UserID, } if err := c.Conn.InsertSessionStart(sessionID, c.sessions[ sessionID ]); err != nil { c.sessions[ sessionID ] = nil diff --git a/backend/pkg/db/types/session.go b/backend/pkg/db/types/session.go index 0205f76f3..d354b0cd2 100644 --- a/backend/pkg/db/types/session.go +++ b/backend/pkg/db/types/session.go @@ -16,7 +16,7 @@ type Session struct { PagesCount int EventsCount int ErrorsCount int - UserID *string + UserID string // pointer?? UserAnonymousID *string Metadata1 *string Metadata2 *string From b4f441132ff63672392168e23312b4e3bcc5c598 Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Sun, 23 Jan 2022 19:38:37 +0100 Subject: [PATCH 187/206] feat-fix (backend): insert userID on SessionStart (3) --- backend/pkg/db/cache/messages_common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/db/cache/messages_common.go b/backend/pkg/db/cache/messages_common.go index c05422cb2..dcf860835 100644 --- a/backend/pkg/db/cache/messages_common.go +++ b/backend/pkg/db/cache/messages_common.go @@ -38,7 +38,7 @@ func (c *PGCache) InsertUserID(sessionID uint64, userID *IOSUserID) error { if err != nil { return err } - session.UserID = &userID.Value + session.UserID = userID.Value return nil } From 8aca491f0e8b9de677b25be0cd028b54383c188a Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 23 Jan 2022 22:14:44 +0100 Subject: [PATCH 188/206] Delete README.md --- scripts/helmcharts/README.md | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 scripts/helmcharts/README.md diff --git a/scripts/helmcharts/README.md b/scripts/helmcharts/README.md deleted file mode 100644 index 2e4475720..000000000 --- a/scripts/helmcharts/README.md +++ /dev/null @@ -1,4 +0,0 @@ -## Installation -helm upgrade --install databases ./databases -n db --create-namespace --wait -f ./values.yaml --atomic - -helm upgrade --install openreplay ./openreplay -n app --create-namespace --wait -f ./values.yaml --atomic From 2470ff1f0a93c735dca66dcf5607c0ed32ead8e4 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Sun, 23 Jan 2022 22:20:41 +0100 Subject: [PATCH 189/206] Update env.js --- frontend/env.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/env.js b/frontend/env.js index 10513c3b1..75f421314 100644 --- a/frontend/env.js +++ b/frontend/env.js @@ -21,7 +21,7 @@ const oss = { MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, ICE_SERVERS: process.env.ICE_SERVERS, - TRACKER_VERSION: '3.4.16', // trackerInfo.version, + TRACKER_VERSION: '3.4.17', // trackerInfo.version, } module.exports = { From 54de370b3191c06cd0354cfa65543fe2bac31b77 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Sun, 23 Jan 2022 23:30:53 +0100 Subject: [PATCH 190/206] feat(nginx): fixed comment in vars.yaml --- scripts/helmcharts/vars.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 6edb0f32c..acae47824 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -106,6 +106,6 @@ chalice: # # and copied to ../openreplay/files/ # sslKey: site.key # sslCert: site.crt +# # Redirecting http to https # customServerConfigs: | -# # Redirecting http to https # return 301 https://$host$request_uri; From e39b7b5c97655ab558113a740b60952b29544d0b Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 24 Jan 2022 01:03:38 +0100 Subject: [PATCH 191/206] changed clickhouse in vars.yaml --- scripts/helmcharts/vars.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index acae47824..14561985d 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -16,7 +16,7 @@ postgresql: &postgres # memory: 3000Mi # cpu: 2 -clickhouse: {} +clickhouse: &clickhouse # For enterpriseEdition # enabled: true From 64351ffe40d803c518f09b95118d7d58c9e8e0e3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Mon, 24 Jan 2022 14:39:07 +0100 Subject: [PATCH 192/206] feat(api): flat search: support empty events --- api/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/schemas.py b/api/schemas.py index d66b92444..af2034061 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -561,7 +561,7 @@ class FlatSessionsSearchPayloadSchema(SessionsSearchPayloadSchema): n_filters = [] n_events = [] for v in values.get("filters", []): - if v["isEvent"]: + if v.get("isEvent"): n_events.append(v) else: n_filters.append(v) From dff39cee656bdeb36a7e48c3d18843fc63a89103 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 25 Jan 2022 12:05:16 +0100 Subject: [PATCH 193/206] feat(api): sessions search default sort feat(api): funnels ignore empty values feat(api): sessions search ignore empty values feat(api): saved search return filter feat(api): saved search return empty filter --- api/chalicelib/core/saved_search.py | 2 ++ api/chalicelib/core/sessions.py | 4 ++++ api/chalicelib/core/significance.py | 2 ++ api/routers/core.py | 2 +- api/schemas.py | 2 +- 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/api/chalicelib/core/saved_search.py b/api/chalicelib/core/saved_search.py index 732fc1596..d1e8fe15f 100644 --- a/api/chalicelib/core/saved_search.py +++ b/api/chalicelib/core/saved_search.py @@ -77,6 +77,8 @@ def get_all(project_id, user_id, details=False): for row in rows: row["createdAt"] = TimeUTC.datetime_to_timestamp(row["createdAt"]) if details: + if isinstance(row["filter"], list) and len(row["filter"]) == 0: + row["filter"] = {} row["filter"] = helper.old_search_payload_to_flat(row["filter"]) return rows diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 8eb58a29d..f5617789d 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -297,6 +297,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr op = __get_sql_operator(f.operator) \ if filter_type not in [schemas.FilterType.events_count] else f.operator is_any = _isAny_opreator(f.operator) + if not is_any and len(f.value) == 0: + continue is_not = False if __is_negation_operator(f.operator): is_not = True @@ -445,6 +447,8 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr is_any = _isAny_opreator(event.operator) if not isinstance(event.value, list): event.value = [event.value] + if not is_any and len(event.value) == 0: + continue op = __get_sql_operator(event.operator) is_not = False if __is_negation_operator(event.operator): diff --git a/api/chalicelib/core/significance.py b/api/chalicelib/core/significance.py index f2261ce59..2580a7584 100644 --- a/api/chalicelib/core/significance.py +++ b/api/chalicelib/core/significance.py @@ -130,6 +130,8 @@ def get_stages_and_events(filter_d, project_id) -> List[RealDictRow]: if not isinstance(s["value"], list): s["value"] = [s["value"]] is_any = sessions._isAny_opreator(s["operator"]) + if not is_any and isinstance(s["value"], list) and len(s["value"]) == 0: + continue op = sessions.__get_sql_operator(s["operator"]) event_type = s["type"].upper() if event_type == events.event_type.CLICK.ui_type: diff --git a/api/routers/core.py b/api/routers/core.py index bbeb30bcd..81ed07cca 100644 --- a/api/routers/core.py +++ b/api/routers/core.py @@ -1124,7 +1124,7 @@ def add_saved_search(projectId: int, data: schemas.SavedSearchSchema = Body(...) @app.get('/{projectId}/saved_search', tags=["savedSearch"]) def get_saved_searches(projectId: int, context: schemas.CurrentContext = Depends(OR_context)): - return {"data": saved_search.get_all(project_id=projectId, user_id=context.user_id)} + return {"data": saved_search.get_all(project_id=projectId, user_id=context.user_id, details=True)} @app.get('/{projectId}/saved_search/{search_id}', tags=["savedSearch"]) diff --git a/api/schemas.py b/api/schemas.py index af2034061..0c3493121 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -536,7 +536,7 @@ class SessionsSearchPayloadSchema(BaseModel): # rangeValue:str=Field(...) startDate: int = Field(None) endDate: int = Field(None) - sort: str = Field(...) + sort: str = Field(default="startTs") order: str = Field(default="DESC") # platform: Optional[PlatformType] = Field(None) events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then) From be61e81c579d3aed21b114321bbf3cf091814cbd Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 24 Jan 2022 23:17:03 +0530 Subject: [PATCH 194/206] chore(nginx): enabling keep alive for frontend Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/nginx-ingress/templates/configMap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml index 77100c35c..05e9a303d 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/configMap.yaml @@ -84,6 +84,8 @@ data: location / { index /index.html; rewrite ^((?!.(js|css|png|svg|jpg|woff|woff2)).)*$ /frontend/index.html break; + proxy_http_version 1.1; + proxy_set_header Connection ""; include /etc/nginx/conf.d/compression.conf; proxy_set_header Host $http_host; proxy_pass http://minio.db.svc.cluster.local:9000/frontend/; From 598243721ca0ada35ce850909dbe67866764f010 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Tue, 25 Jan 2022 20:10:35 +0530 Subject: [PATCH 195/206] chore(helm): comment for shared persistence. Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/helmcharts/vars.yaml b/scripts/helmcharts/vars.yaml index 14561985d..ca165eda3 100644 --- a/scripts/helmcharts/vars.yaml +++ b/scripts/helmcharts/vars.yaml @@ -71,6 +71,17 @@ global: enterpriseEditionLicense: "" domainName: "" +# If there is multiple nodes in the kubernetes cluster, +# we'll have to create a NFS share PVC for both the containers to share data. +# If it's the single node, we'll use hostVolume, which is default for community installation. +# Note: Both PVC name should be same. +# sink: +# pvc: +# name: mysharedpersistence +# storage: +# pvc: +# name: mysharedpersistence + chalice: env: jwt_secret: "SetARandomStringHere" From 229155997396f292a90d647bb6c0b48fa45d6dce Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Tue, 25 Jan 2022 20:11:38 +0530 Subject: [PATCH 196/206] chore(helm): use externalTrafficPolicy only for lb/nodeport type svc. Signed-off-by: Rajesh Rajendran --- .../openreplay/charts/nginx-ingress/templates/service.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml index 94c034375..35d6d969c 100644 --- a/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml +++ b/scripts/helmcharts/openreplay/charts/nginx-ingress/templates/service.yaml @@ -6,8 +6,10 @@ metadata: {{- include "nginx-ingress.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} + {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort")}} # Make sure to get client ip externalTrafficPolicy: Local + {{- end}} ports: {{- range .Values.service.ports }} - port: {{ .port }} From 8ee4889a5e5883745c676a9f64502efad94b2469 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 25 Jan 2022 16:56:52 +0100 Subject: [PATCH 197/206] feat(api): sessions search fixed isNot index feat(api): sessions search by metadata use source instead of key --- api/chalicelib/core/sessions.py | 8 ++++---- api/schemas.py | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index f5617789d..113c2c754 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -384,12 +384,12 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr meta_keys = metadata.get(project_id=project_id) meta_keys = {m["key"]: m["index"] for m in meta_keys} # op = __get_sql_operator(f.operator) - if f.key in meta_keys.keys(): + if f.source in meta_keys.keys(): extra_constraints.append( - _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + _multiple_conditions(f"s.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) ss_constraints.append( - _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.key])} {op} %({f_k})s", + _multiple_conditions(f"ms.{metadata.index_to_colname(meta_keys[f.source])} {op} %({f_k})s", f.value, is_not=is_not, value_key=f_k)) elif filter_type in [schemas.FilterType.user_id, schemas.FilterType.user_id_ios]: # op = __get_sql_operator(f.operator) @@ -682,7 +682,7 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr AND start_ts >= %(startDate)s AND start_ts <= %(endDate)s AND duration IS NOT NULL - ) {"" if or_events else ("AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ + ) {"" if or_events else (f"AS event_{event_index}" + ("ON(TRUE)" if event_index > 0 else ""))}\ """) else: events_query_from.append(f"""\ diff --git a/api/schemas.py b/api/schemas.py index 0c3493121..a09178df9 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -502,16 +502,18 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): class _SessionSearchFilterSchema(__MixedSearchFilter): is_event: bool = Field(False, const=False) custom: Optional[List[str]] = Field(None) - key: Optional[str] = Field(None) value: Union[Optional[Union[IssueType, PlatformType, int, str]], Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...) type: FilterType = Field(...) operator: Union[SearchEventOperator, MathOperator] = Field(...) - source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) + source: Optional[Union[ErrorSource, str]] = Field(default=ErrorSource.js_exception) @root_validator def filter_validator(cls, values): - if values.get("type") == FilterType.issue: + if values.get("type") == FilterType.metadata: + assert values.get("source") is not None and len(values["source"]) > 0, \ + "must specify a valid 'source' for metadata filter" + elif values.get("type") == FilterType.issue: for v in values.get("value"): assert isinstance(v, IssueType), f"value should be of type IssueType for {values.get('type')} filter" elif values.get("type") == FilterType.platform: From fc396d792a57bcce738cf4ef370030c1c341d7d6 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Tue, 25 Jan 2022 17:07:39 +0100 Subject: [PATCH 198/206] feat(api): sessions search use source and sourceOperator instead of custom and customOperator feat(api): sessions search remove custom, customOperator, key --- api/chalicelib/core/sessions.py | 30 +++++++++++++++--------------- api/schemas.py | 20 +++++++------------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/api/chalicelib/core/sessions.py b/api/chalicelib/core/sessions.py index 113c2c754..5f7cd23af 100644 --- a/api/chalicelib/core/sessions.py +++ b/api/chalicelib/core/sessions.py @@ -488,10 +488,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_where.append( _multiple_conditions(f"main.{events.event_type.INPUT.column} {op} %({e_k})s", event.value, value_key=e_k)) - if event.custom is not None and len(event.custom) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, value_key=f"custom{i}")) - full_args = {**full_args, **_multiple_values(event.custom, value_key=f"custom{i}")} + full_args = {**full_args, **_multiple_values(event.source, value_key=f"custom{i}")} elif event_type == events.event_type.LOCATION.ui_type: event_from = event_from % f"{events.event_type.LOCATION.table} AS main " @@ -551,10 +551,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event_where.append( _multiple_conditions(f"main.{events.event_type.INPUT_IOS.column} {op} %({e_k})s", event.value, value_key=e_k)) - if event.custom is not None and len(event.custom) > 0: - event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.custom, + if event.source is not None and len(event.source) > 0: + event_where.append(_multiple_conditions(f"main.value ILIKE %(custom{i})s", event.source, value_key="custom{i}")) - full_args = {**full_args, **_multiple_values(event.custom, f"custom{i}")} + full_args = {**full_args, **_multiple_values(event.source, f"custom{i}")} elif event_type == events.event_type.VIEW_IOS.ui_type: event_from = event_from % f"{events.event_type.VIEW_IOS.table} AS main " if not is_any: @@ -598,10 +598,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr # colname = col["column"] # tname = "main" # e_k += "_custom" - # full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + # full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} # event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - # _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - # event.custom, value_key=e_k)) + # _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + # event.source, value_key=e_k)) elif event_type in [schemas.PerformanceEventType.location_dom_complete, schemas.PerformanceEventType.location_largest_contentful_paint_time, schemas.PerformanceEventType.location_ttfb, @@ -622,11 +622,11 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr _multiple_conditions(f"main.{events.event_type.LOCATION.column} {op} %({e_k})s", event.value, value_key=e_k)) e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} event_where.append(f"{tname}.{colname} IS NOT NULL AND {tname}.{colname}>0 AND " + - _multiple_conditions(f"{tname}.{colname} {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) + _multiple_conditions(f"{tname}.{colname} {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) elif event_type == schemas.PerformanceEventType.time_between_events: event_from = event_from % f"{getattr(events.event_type, event.value[0].type).table} AS main INNER JOIN {getattr(events.event_type, event.value[1].type).table} AS main2 USING(session_id) " if not isinstance(event.value[0].value, list): @@ -657,10 +657,10 @@ def search_query_parts(data, error_status, errors_only, favorite_only, issue, pr event.value[1].value, value_key=e_k2)) e_k += "_custom" - full_args = {**full_args, **_multiple_values(event.custom, value_key=e_k)} + full_args = {**full_args, **_multiple_values(event.source, value_key=e_k)} event_where.append( - _multiple_conditions(f"main2.timestamp - main.timestamp {event.customOperator} %({e_k})s", - event.custom, value_key=e_k)) + _multiple_conditions(f"main2.timestamp - main.timestamp {event.sourceOperator} %({e_k})s", + event.source, value_key=e_k)) else: diff --git a/api/schemas.py b/api/schemas.py index a09178df9..11d882e79 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -467,22 +467,20 @@ class __MixedSearchFilter(BaseModel): class _SessionSearchEventRaw(__MixedSearchFilter): is_event: bool = Field(True, const=True) - custom: Optional[List[Union[int, str]]] = Field(None, min_items=1) - customOperator: Optional[MathOperator] = Field(None) - key: Optional[str] = Field(None) value: Union[str, List[str]] = Field(...) type: Union[EventType, PerformanceEventType] = Field(...) operator: SearchEventOperator = Field(...) - source: Optional[ErrorSource] = Field(default=ErrorSource.js_exception) + source: Optional[Union[ErrorSource,List[Union[int, str]]]] = Field(default=ErrorSource.js_exception) + sourceOperator: Optional[MathOperator] = Field(None) @root_validator def event_validator(cls, values): if isinstance(values.get("type"), PerformanceEventType): if values.get("type") == PerformanceEventType.fetch_failed: return values - assert values.get("custom") is not None, "custom should not be null for PerformanceEventType" - assert values.get("customOperator") is not None \ - , "customOperator should not be null for PerformanceEventType" + assert values.get("source") is not None, "source should not be null for PerformanceEventType" + assert values.get("sourceOperator") is not None \ + , "sourceOperator should not be null for PerformanceEventType" if values["type"] == PerformanceEventType.time_between_events: assert len(values.get("value", [])) == 2, \ f"must provide 2 Events as value for {PerformanceEventType.time_between_events}" @@ -490,8 +488,8 @@ class _SessionSearchEventRaw(__MixedSearchFilter): and isinstance(values["value"][1], _SessionSearchEventRaw) \ , f"event should be of type _SessionSearchEventRaw for {PerformanceEventType.time_between_events}" else: - for c in values["custom"]: - assert isinstance(c, int), f"custom value should be of type int for {values.get('type')}" + for c in values["source"]: + assert isinstance(c, int), f"source value should be of type int for {values.get('type')}" return values @@ -501,7 +499,6 @@ class _SessionSearchEventSchema(_SessionSearchEventRaw): class _SessionSearchFilterSchema(__MixedSearchFilter): is_event: bool = Field(False, const=False) - custom: Optional[List[str]] = Field(None) value: Union[Optional[Union[IssueType, PlatformType, int, str]], Optional[List[Union[IssueType, PlatformType, int, str]]]] = Field(...) type: FilterType = Field(...) @@ -534,13 +531,10 @@ class _SessionSearchFilterSchema(__MixedSearchFilter): class SessionsSearchPayloadSchema(BaseModel): events: List[_SessionSearchEventSchema] = Field([]) filters: List[_SessionSearchFilterSchema] = Field([]) - # custom:dict=Field(...) - # rangeValue:str=Field(...) startDate: int = Field(None) endDate: int = Field(None) sort: str = Field(default="startTs") order: str = Field(default="DESC") - # platform: Optional[PlatformType] = Field(None) events_order: Optional[SearchEventOrder] = Field(default=SearchEventOrder._then) class Config: From 7051820a66fa93a56b5c30c18630261efe158a48 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Tue, 25 Jan 2022 22:10:49 +0530 Subject: [PATCH 199/206] chore(helm): removing quotes from deployment. --- .../openreplay/charts/alerts/templates/deployment.yaml | 4 ++-- .../openreplay/charts/assets/templates/deployment.yaml | 8 ++++---- .../charts/chalice/templates/deployment.yaml | 10 +++++----- .../openreplay/charts/http/templates/deployment.yaml | 6 +++--- .../charts/storage/templates/deployment.yaml | 8 ++++---- .../charts/utilities/templates/deployment.yaml | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml index 6cdbe17b6..f88266e66 100644 --- a/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/alerts/templates/deployment.yaml @@ -59,9 +59,9 @@ spec: value: '{{ .Values.global.s3.endpoint }}' {{- end }} - name: S3_KEY - value: '{{ .Values.global.s3.accessKey }}' + value: {{ .Values.global.s3.accessKey }} - name: S3_SECRET - value: '{{ .Values.global.s3.secretKey }}' + value: {{ .Values.global.s3.secretKey }} - name: AWS_DEFAULT_REGION value: '{{ .Values.global.s3.region }}' - name: EMAIL_HOST diff --git a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml index 5df3ee4a8..1af5164ca 100644 --- a/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/assets/templates/deployment.yaml @@ -39,11 +39,11 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: AWS_ACCESS_KEY_ID - value: '{{ .Values.global.s3.accessKey }}' + value: {{ .Values.global.s3.accessKey }} - name: AWS_SECRET_ACCESS_KEY - value: '{{ .Values.global.s3.secretKey }}' + value: {{ .Values.global.s3.secretKey }} - name: S3_BUCKET_ASSETS - value: '{{ .Values.global.s3.assetsBucket }}' + value: {{ .Values.global.s3.assetsBucket }} - name: LICENSE_KEY value: '{{ .Values.global.enterpriseEditionLicense }}' - name: AWS_ENDPOINT @@ -59,7 +59,7 @@ spec: # Ref: https://stackoverflow.com/questions/53634583/go-template-split-string-by-delimiter # We need https://bucketname.s3endpoint - name: ASSETS_ORIGIN - value: '{{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }}' + value: {{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml index 25ec6c387..f3eb46d1d 100644 --- a/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/chalice/templates/deployment.yaml @@ -61,19 +61,19 @@ spec: value: '{{ .Values.global.s3.endpoint }}' {{- end}} - name: S3_KEY - value: '{{ .Values.global.s3.accessKey }}' + value: {{ .Values.global.s3.accessKey }} - name: S3_SECRET - value: '{{ .Values.global.s3.secretKey }}' + value: {{ .Values.global.s3.secretKey }} - name: AWS_DEFAULT_REGION value: '{{ .Values.global.s3.region }}' - name: sessions_region value: '{{ .Values.global.s3.region }}' - name: sessions_bucket - value: '{{ .Values.global.s3.recordingsBucket }}' + value: {{ .Values.global.s3.recordingsBucket }} - name: sourcemaps_bucket - value: '{{ .Values.global.s3.sourcemapsBucket }}' + value: {{ .Values.global.s3.sourcemapsBucket }} - name: js_cache_bucket - value: '{{ .Values.global.s3.assetsBucket }}' + value: {{ .Values.global.s3.assetsBucket }} - name: EMAIL_HOST value: '{{ .Values.global.email.emailHost }}' - name: EMAIL_PORT diff --git a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml index 7de2a80c5..d728b7d02 100644 --- a/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/http/templates/deployment.yaml @@ -39,9 +39,9 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: AWS_ACCESS_KEY_ID - value: '{{ .Values.global.s3.accessKey }}' + value: {{ .Values.global.s3.accessKey }} - name: AWS_SECRET_ACCESS_KEY - value: '{{ .Values.global.s3.secretKey }}' + value: {{ .Values.global.s3.secretKey }} - name: AWS_REGION value: '{{ .Values.global.s3.region }}' - name: LICENSE_KEY @@ -55,7 +55,7 @@ spec: - name: POSTGRES_STRING value: 'postgres://{{ .Values.global.postgresql.postgresqlUser }}:{{ .Values.global.postgresql.postgresqlPassword }}@{{ .Values.global.postgresql.postgresqlHost }}:{{ .Values.global.postgresql.postgresqlPort }}/{{ .Values.global.postgresql.postgresqlDatabase }}' - name: ASSETS_ORIGIN - value: '{{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }}' + value: {{ (split "://" .Values.global.s3.endpoint)._0 }}://{{.Values.global.s3.assetsBucket}}.{{ (split "://" .Values.global.s3.endpoint)._1 }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' diff --git a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml index c5ec92cf4..0004bfc7f 100644 --- a/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/storage/templates/deployment.yaml @@ -39,9 +39,9 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: AWS_ACCESS_KEY_ID - value: '{{ .Values.global.s3.accessKey }}' + value: {{ .Values.global.s3.accessKey }} - name: AWS_SECRET_ACCESS_KEY - value: '{{ .Values.global.s3.secretKey }}' + value: {{ .Values.global.s3.secretKey }} - name: AWS_ENDPOINT value: '{{ .Values.global.s3.endpoint }}' - name: AWS_REGION_WEB @@ -49,9 +49,9 @@ spec: - name: AWS_REGION_IOS value: '{{ .Values.global.s3.region }}' - name: S3_BUCKET_WEB - value: '{{ .Values.global.s3.recordingsBucket }}' + value: {{ .Values.global.s3.recordingsBucket }} - name: S3_BUCKET_IOS - value: '{{ .Values.global.s3.recordingsBucket }}' + value: {{ .Values.global.s3.recordingsBucket }} - name: REDIS_STRING value: '{{ .Values.global.redis.redisHost }}:{{ .Values.global.redis.redisPort }}' - name: LICENSE_KEY diff --git a/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml b/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml index 628cda58d..a1b005253 100644 --- a/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml +++ b/scripts/helmcharts/openreplay/charts/utilities/templates/deployment.yaml @@ -47,9 +47,9 @@ spec: value: '{{ .Values.global.s3.endpoint }}' {{- end}} - name: S3_KEY - value: "{{ .Values.global.s3.accessKey }}" + value: {{ .Values.global.s3.accessKey }} - name: S3_SECRET - value: "{{ .Values.global.s3.secretKey }}" + value: {{ .Values.global.s3.secretKey }} {{- range $key, $val := .Values.env }} - name: {{ $key }} value: '{{ $val }}' From dc336dcfaf9e2fede5520b90b16d4e390afab171 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Wed, 26 Jan 2022 09:13:36 +0530 Subject: [PATCH 200/206] fix(ansible): duplicate colon Signed-off-by: Rajesh Rajendran --- scripts/helmcharts/vars_template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/helmcharts/vars_template.yaml b/scripts/helmcharts/vars_template.yaml index 84ce36ee6..2230e0a75 100644 --- a/scripts/helmcharts/vars_template.yaml +++ b/scripts/helmcharts/vars_template.yaml @@ -24,14 +24,14 @@ kafka: &kafka # For enterpriseEdition # enabled: true - kafkaHost: "{{ kafka_endpoint }}" + kafkaHost: "{{ kafka_endpoint.split(':')[0] }}" kafkaPort: "{{ kafka_endpoint.split(':')[-1] }}" kafkaUseSsl: "{{ kafka_ssl }}" redis: &redis # For enterpriseEdition # enabled: false - redisHost: "{{ redis_endpoint }}" + redisHost: "{{ redis_endpoint.split(':')[0] }}" redisPort: "{{ redis_endpoint.split(':')[-1] }}" minio: From fe528d795b678ab95c254be23982940f4f36fef5 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 26 Jan 2022 11:44:37 +0100 Subject: [PATCH 201/206] feat(DB): EE init_schema check for all tables --- .../db/init_dbs/postgresql/init_schema.sql | 1061 ++++++++++------- 1 file changed, 605 insertions(+), 456 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 35f3a5dd4..7db4b0bed 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1,7 +1,9 @@ BEGIN; --- Schemas and functions definitions: + CREATE SCHEMA IF NOT EXISTS events_common; CREATE SCHEMA IF NOT EXISTS events; +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS pgcrypto; CREATE OR REPLACE FUNCTION openreplay_version() RETURNS text AS @@ -9,7 +11,6 @@ $$ SELECT 'v1.4.0-ee' $$ LANGUAGE sql IMMUTABLE; --- --- accounts.sql --- CREATE OR REPLACE FUNCTION generate_api_key(length integer) RETURNS text AS $$ @@ -29,7 +30,7 @@ begin end; $$ LANGUAGE plpgsql; --- --- events.sql --- + CREATE OR REPLACE FUNCTION events.funnel(steps integer[], m integer) RETURNS boolean AS $$ @@ -54,13 +55,14 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; --- --- integrations.sql --- + CREATE OR REPLACE FUNCTION notify_integration() RETURNS trigger AS $$ BEGIN IF NEW IS NULL THEN - PERFORM pg_notify('integration', (row_to_json(OLD)::text || '{"options": null, "request_data": null}'::text)); + PERFORM pg_notify('integration', + (row_to_json(OLD)::text || '{"options": null, "request_data": null}'::text)); ELSIF (OLD IS NULL) OR (OLD.options <> NEW.options) THEN PERFORM pg_notify('integration', row_to_json(NEW)::text); END IF; @@ -68,7 +70,7 @@ BEGIN END; $$ LANGUAGE plpgsql; --- --- alerts.sql --- + CREATE OR REPLACE FUNCTION notify_alert() RETURNS trigger AS $$ @@ -85,7 +87,6 @@ BEGIN END ; $$ LANGUAGE plpgsql; --- --- projects.sql --- CREATE OR REPLACE FUNCTION notify_project() RETURNS trigger AS $$ @@ -95,28 +96,51 @@ BEGIN END; $$ LANGUAGE plpgsql; --- All tables and types: + DO $$ BEGIN - IF EXISTS(SELECT - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_name = 'tenants') THEN - raise notice 'DB exists, skipping creation query'; + IF (with to_check (name) as ( + values ('alerts'), + ('announcements'), + ('assigned_sessions'), + ('autocomplete'), + ('basic_authentication'), + ('errors'), + ('funnels'), + ('integrations'), + ('issues'), + ('jira_cloud'), + ('jobs'), + ('metric_series'), + ('metrics'), + ('notifications'), + ('oauth_authentication'), + ('projects'), + ('roles'), + ('roles_projects'), + ('searches'), + ('sessions'), + ('tenants'), + ('traces'), + ('user_favorite_errors'), + ('user_favorite_sessions'), + ('user_viewed_errors'), + ('user_viewed_sessions'), + ('users'), + ('webhooks') + ) + select bool_and(exists(select * + from information_schema.tables t + where table_schema = 'public' + AND table_name = to_check.name)) as all_present + from to_check) THEN + raise notice 'All public schema tables exists'; ELSE - raise notice 'Creating DB'; + raise notice 'Some or all public schema tables are missing, creating missing tables'; --- --- public.sql --- - - CREATE EXTENSION IF NOT EXISTS pg_trgm; - CREATE EXTENSION IF NOT EXISTS pgcrypto; - - --- --- accounts.sql --- - - CREATE TABLE tenants + CREATE TABLE IF NOT EXISTS tenants ( tenant_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, user_id text NOT NULL DEFAULT generate_api_key(20), @@ -135,7 +159,7 @@ $$ ); - CREATE TABLE roles + CREATE TABLE IF NOT EXISTS roles ( role_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, @@ -148,9 +172,14 @@ $$ deleted_at timestamp NULL DEFAULT NULL ); - CREATE TYPE user_role AS ENUM ('owner', 'admin', 'member'); + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'user_role') THEN + CREATE TYPE user_role AS ENUM ('owner','admin','member'); + END IF; - CREATE TABLE users + + CREATE TABLE IF NOT EXISTS users ( user_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, @@ -222,16 +251,16 @@ $$ }'::jsonb, api_key text UNIQUE default generate_api_key(20) not null, jwt_iat timestamp without time zone NULL DEFAULT NULL, - data jsonb NOT NULL DEFAULT '{}'::jsonb, + data jsonb NOT NULL DEFAULT'{}'::jsonb, weekly_report boolean NOT NULL DEFAULT TRUE, origin text NULL DEFAULT NULL, role_id integer REFERENCES roles (role_id) ON DELETE SET NULL, internal_id text NULL DEFAULT NULL ); - CREATE INDEX users_tenant_id_deleted_at_N_idx ON users (tenant_id) WHERE deleted_at ISNULL; + CREATE INDEX IF NOT EXISTS users_tenant_id_deleted_at_N_idx ON users (tenant_id) WHERE deleted_at ISNULL; - CREATE TABLE basic_authentication + CREATE TABLE IF NOT EXISTS basic_authentication ( user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, password text DEFAULT NULL, @@ -244,9 +273,13 @@ $$ UNIQUE (user_id) ); + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'oauth_provider') THEN + CREATE TYPE oauth_provider AS ENUM ('jira','github'); + END IF; - CREATE TYPE oauth_provider AS ENUM ('jira', 'github'); - CREATE TABLE oauth_authentication + CREATE TABLE IF NOT EXISTS oauth_authentication ( user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, provider oauth_provider NOT NULL, @@ -256,9 +289,7 @@ $$ ); --- --- projects.sql --- - - CREATE TABLE projects + CREATE TABLE IF NOT EXISTS projects ( project_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_key varchar(20) NOT NULL UNIQUE DEFAULT generate_api_key(20), @@ -279,35 +310,43 @@ $$ metadata_8 text DEFAULT NULL, metadata_9 text DEFAULT NULL, metadata_10 text DEFAULT NULL, - gdpr jsonb NOT NULL DEFAULT '{ + gdpr jsonb NOT NULL DEFAULT'{ "maskEmails": true, "sampleRate": 33, "maskNumbers": false, "defaultInputMode": "plain" - }'::jsonb -- ?????? + }'::jsonb ); - CREATE INDEX projects_project_key_idx ON public.projects (project_key); + + CREATE INDEX IF NOT EXISTS projects_project_key_idx ON public.projects (project_key); + DROP TRIGGER IF EXISTS on_insert_or_update ON projects; CREATE TRIGGER on_insert_or_update AFTER INSERT OR UPDATE ON projects FOR EACH ROW EXECUTE PROCEDURE notify_project(); - CREATE TABLE roles_projects + CREATE TABLE IF NOT EXISTS roles_projects ( role_id integer NOT NULL REFERENCES roles (role_id) ON DELETE CASCADE, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, CONSTRAINT roles_projects_pkey PRIMARY KEY (role_id, project_id) ); - CREATE INDEX roles_projects_role_id_idx ON roles_projects (role_id); - CREATE INDEX roles_projects_project_id_idx ON roles_projects (project_id); + CREATE INDEX IF NOT EXISTS roles_projects_role_id_idx ON roles_projects (role_id); + CREATE INDEX IF NOT EXISTS roles_projects_project_id_idx ON roles_projects (project_id); --- --- webhooks.sql --- - create type webhook_type as enum ('webhook', 'slack', 'email'); + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'webhook_type') THEN + create type webhook_type as enum ('webhook','slack','email'); + END IF; + END; - create table webhooks + + create table IF NOT EXISTS webhooks ( webhook_id integer generated by default as identity constraint webhooks_pkey @@ -325,10 +364,8 @@ $$ name varchar(100) ); --- --- notifications.sql --- - - CREATE TABLE notifications + CREATE TABLE IF NOT EXISTS notifications ( notification_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, tenant_id integer REFERENCES tenants (tenant_id) ON DELETE CASCADE, @@ -339,25 +376,24 @@ $$ button_url text NULL, image_url text NULL, created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - options jsonb NOT NULL DEFAULT '{}'::jsonb, + options jsonb NOT NULL DEFAULT'{}'::jsonb, CONSTRAINT notification_tenant_xor_user CHECK ( tenant_id NOTNULL AND user_id ISNULL OR tenant_id ISNULL AND user_id NOTNULL ) ); - CREATE INDEX notifications_user_id_index ON notifications (user_id); - CREATE INDEX notifications_tenant_id_index ON notifications (tenant_id); - CREATE INDEX notifications_created_at_index ON notifications (created_at DESC); - CREATE INDEX notifications_created_at_epoch_idx ON notifications (CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) DESC); + CREATE INDEX IF NOT EXISTS notifications_user_id_index ON notifications (user_id); + CREATE INDEX IF NOT EXISTS notifications_tenant_id_index ON notifications (tenant_id); + CREATE INDEX IF NOT EXISTS notifications_created_at_index ON notifications (created_at DESC); + CREATE INDEX IF NOT EXISTS notifications_created_at_epoch_idx ON notifications (CAST(EXTRACT(EPOCH FROM created_at) * 1000 AS BIGINT) DESC); - CREATE TABLE user_viewed_notifications + CREATE TABLE IF NOT EXISTS user_viewed_notifications ( user_id integer NOT NULL REFERENCES users (user_id) on delete cascade, notification_id integer NOT NULL REFERENCES notifications (notification_id) on delete cascade, constraint user_viewed_notifications_pkey primary key (user_id, notification_id) ); --- --- funnels.sql --- - CREATE TABLE funnels + CREATE TABLE IF NOT EXISTS funnels ( funnel_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -369,14 +405,18 @@ $$ is_public boolean NOT NULL DEFAULT False ); - CREATE INDEX funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); - CREATE INDEX funnels_project_id_idx ON public.funnels (project_id); + CREATE INDEX IF NOT EXISTS funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); + CREATE INDEX IF NOT EXISTS funnels_project_id_idx ON public.funnels (project_id); --- --- announcements.sql --- + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'announcement_type') THEN + create type announcement_type as enum ('notification','alert'); + END IF; + END; - create type announcement_type as enum ('notification', 'alert'); - - create table announcements + create table IF NOT EXISTS announcements ( announcement_id serial not null constraint announcements_pk @@ -390,18 +430,25 @@ $$ type announcement_type default 'notification'::announcement_type not null ); --- --- integrations.sql --- + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'integration_provider') THEN + CREATE TYPE integration_provider AS ENUM ('bugsnag','cloudwatch','datadog','newrelic','rollbar','sentry','stackdriver','sumologic','elasticsearch'); --,'jira','github'); + END IF; + END; - CREATE TYPE integration_provider AS ENUM ('bugsnag', 'cloudwatch', 'datadog', 'newrelic', 'rollbar', 'sentry', 'stackdriver', 'sumologic', 'elasticsearch'); --, 'jira', 'github'); - CREATE TABLE integrations + CREATE TABLE IF NOT EXISTS integrations ( project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, provider integration_provider NOT NULL, options jsonb NOT NULL, - request_data jsonb NOT NULL DEFAULT '{}'::jsonb, + request_data jsonb NOT NULL DEFAULT'{}'::jsonb, PRIMARY KEY (project_id, provider) ); + DROP TRIGGER IF EXISTS on_insert_or_update_or_delete ON integrations; + CREATE TRIGGER on_insert_or_update_or_delete AFTER INSERT OR UPDATE OR DELETE ON integrations @@ -409,7 +456,7 @@ $$ EXECUTE PROCEDURE notify_integration(); - create table jira_cloud + CREATE TABLE IF NOT EXISTS jira_cloud ( user_id integer not null constraint jira_cloud_pk @@ -422,32 +469,35 @@ $$ url text ); + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'issue_type') THEN + CREATE TYPE issue_type AS ENUM ( + 'click_rage', + 'dead_click', + 'excessive_scrolling', + 'bad_request', + 'missing_resource', + 'memory', + 'cpu', + 'slow_resource', + 'slow_page_load', + 'crash', + 'ml_cpu', + 'ml_memory', + 'ml_dead_click', + 'ml_click_rage', + 'ml_mouse_thrashing', + 'ml_excessive_scrolling', + 'ml_slow_resources', + 'custom', + 'js_exception' + ); + END IF; + END; --- --- issues.sql --- - - CREATE TYPE issue_type AS ENUM ( - 'click_rage', - 'dead_click', - 'excessive_scrolling', - 'bad_request', - 'missing_resource', - 'memory', - 'cpu', - 'slow_resource', - 'slow_page_load', - 'crash', - 'ml_cpu', - 'ml_memory', - 'ml_dead_click', - 'ml_click_rage', - 'ml_mouse_thrashing', - 'ml_excessive_scrolling', - 'ml_slow_resources', - 'custom', - 'js_exception' - ); - - CREATE TABLE issues + CREATE TABLE IF NOT EXISTS issues ( issue_id text NOT NULL PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -455,16 +505,26 @@ $$ context_string text NOT NULL, context jsonb DEFAULT NULL ); - CREATE INDEX issues_issue_id_type_idx ON issues (issue_id, type); - CREATE INDEX issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); - CREATE INDEX issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); - CREATE INDEX issues_project_id_idx ON issues (project_id); + CREATE INDEX IF NOT EXISTS issues_issue_id_type_idx ON issues (issue_id, type); + CREATE INDEX IF NOT EXISTS issues_context_string_gin_idx ON public.issues USING GIN (context_string gin_trgm_ops); + CREATE INDEX IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); + CREATE INDEX IF NOT EXISTS issues_project_id_idx ON issues (project_id); --- --- errors.sql --- - - CREATE TYPE error_source AS ENUM ('js_exception', 'bugsnag', 'cloudwatch', 'datadog', 'newrelic', 'rollbar', 'sentry', 'stackdriver', 'sumologic'); - CREATE TYPE error_status AS ENUM ('unresolved', 'resolved', 'ignored'); - CREATE TABLE errors + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'error_source') THEN + CREATE TYPE error_source AS ENUM ('js_exception','bugsnag','cloudwatch','datadog','newrelic','rollbar','sentry','stackdriver','sumologic'); + END IF; + END; + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'error_status') THEN + CREATE TYPE error_status AS ENUM ('unresolved','resolved','ignored'); + END IF; + END; + CREATE TABLE IF NOT EXISTS errors ( error_id text NOT NULL PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -477,40 +537,54 @@ $$ stacktrace jsonb, --to save the stacktrace and not query S3 another time stacktrace_parsed_at timestamp ); - CREATE INDEX errors_project_id_source_idx ON errors (project_id, source); - CREATE INDEX errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); - CREATE INDEX errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); - CREATE INDEX errors_project_id_idx ON public.errors (project_id); - CREATE INDEX errors_project_id_status_idx ON public.errors (project_id, status); - CREATE INDEX errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; - CREATE INDEX errors_project_id_error_id_idx ON public.errors (project_id, error_id); - CREATE INDEX errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; - CREATE INDEX errors_error_id_idx ON errors (error_id); - CREATE INDEX errors_parent_error_id_idx ON errors (parent_error_id); + CREATE INDEX IF NOT EXISTS errors_project_id_source_idx ON errors (project_id, source); + CREATE INDEX IF NOT EXISTS errors_message_gin_idx ON public.errors USING GIN (message gin_trgm_ops); + CREATE INDEX IF NOT EXISTS errors_name_gin_idx ON public.errors USING GIN (name gin_trgm_ops); + CREATE INDEX IF NOT EXISTS errors_project_id_idx ON public.errors (project_id); + CREATE INDEX IF NOT EXISTS errors_project_id_status_idx ON public.errors (project_id, status); + CREATE INDEX IF NOT EXISTS errors_project_id_error_id_js_exception_idx ON public.errors (project_id, error_id) WHERE source = 'js_exception'; + CREATE INDEX IF NOT EXISTS errors_project_id_error_id_idx ON public.errors (project_id, error_id); + CREATE INDEX IF NOT EXISTS errors_project_id_error_id_integration_idx ON public.errors (project_id, error_id) WHERE source != 'js_exception'; + CREATE INDEX IF NOT EXISTS errors_error_id_idx ON errors (error_id); + CREATE INDEX IF NOT EXISTS errors_parent_error_id_idx ON errors (parent_error_id); - CREATE TABLE user_favorite_errors + CREATE TABLE IF NOT EXISTS user_favorite_errors ( user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, PRIMARY KEY (user_id, error_id) ); - CREATE TABLE user_viewed_errors + CREATE TABLE IF NOT EXISTS user_viewed_errors ( user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, PRIMARY KEY (user_id, error_id) ); - CREATE INDEX user_viewed_errors_user_id_idx ON public.user_viewed_errors (user_id); - CREATE INDEX user_viewed_errors_error_id_idx ON public.user_viewed_errors (error_id); - - --- --- sessions.sql --- - CREATE TYPE device_type AS ENUM ('desktop', 'tablet', 'mobile', 'other'); - CREATE TYPE country AS ENUM ('UN', 'RW', 'SO', 'YE', 'IQ', 'SA', 'IR', 'CY', 'TZ', 'SY', 'AM', 'KE', 'CD', 'DJ', 'UG', 'CF', 'SC', 'JO', 'LB', 'KW', 'OM', 'QA', 'BH', 'AE', 'IL', 'TR', 'ET', 'ER', 'EG', 'SD', 'GR', 'BI', 'EE', 'LV', 'AZ', 'LT', 'SJ', 'GE', 'MD', 'BY', 'FI', 'AX', 'UA', 'MK', 'HU', 'BG', 'AL', 'PL', 'RO', 'XK', 'ZW', 'ZM', 'KM', 'MW', 'LS', 'BW', 'MU', 'SZ', 'RE', 'ZA', 'YT', 'MZ', 'MG', 'AF', 'PK', 'BD', 'TM', 'TJ', 'LK', 'BT', 'IN', 'MV', 'IO', 'NP', 'MM', 'UZ', 'KZ', 'KG', 'TF', 'HM', 'CC', 'PW', 'VN', 'TH', 'ID', 'LA', 'TW', 'PH', 'MY', 'CN', 'HK', 'BN', 'MO', 'KH', 'KR', 'JP', 'KP', 'SG', 'CK', 'TL', 'RU', 'MN', 'AU', 'CX', 'MH', 'FM', 'PG', 'SB', 'TV', 'NR', 'VU', 'NC', 'NF', 'NZ', 'FJ', 'LY', 'CM', 'SN', 'CG', 'PT', 'LR', 'CI', 'GH', 'GQ', 'NG', 'BF', 'TG', 'GW', 'MR', 'BJ', 'GA', 'SL', 'ST', 'GI', 'GM', 'GN', 'TD', 'NE', 'ML', 'EH', 'TN', 'ES', 'MA', 'MT', 'DZ', 'FO', 'DK', 'IS', 'GB', 'CH', 'SE', 'NL', 'AT', 'BE', 'DE', 'LU', 'IE', 'MC', 'FR', 'AD', 'LI', 'JE', 'IM', 'GG', 'SK', 'CZ', 'NO', 'VA', 'SM', 'IT', 'SI', 'ME', 'HR', 'BA', 'AO', 'NA', 'SH', 'BV', 'BB', 'CV', 'GY', 'GF', 'SR', 'PM', 'GL', 'PY', 'UY', 'BR', 'FK', 'GS', 'JM', 'DO', 'CU', 'MQ', 'BS', 'BM', 'AI', 'TT', 'KN', 'DM', 'AG', 'LC', 'TC', 'AW', 'VG', 'VC', 'MS', 'MF', 'BL', 'GP', 'GD', 'KY', 'BZ', 'SV', 'GT', 'HN', 'NI', 'CR', 'VE', 'EC', 'CO', 'PA', 'HT', 'AR', 'CL', 'BO', 'PE', 'MX', 'PF', 'PN', 'KI', 'TK', 'TO', 'WF', 'WS', 'NU', 'MP', 'GU', 'PR', 'VI', 'UM', 'AS', 'CA', 'US', 'PS', 'RS', 'AQ', 'SX', 'CW', 'BQ', 'SS'); - CREATE TYPE platform AS ENUM ('web','ios','android'); - - CREATE TABLE sessions + CREATE INDEX IF NOT EXISTS user_viewed_errors_user_id_idx ON public.user_viewed_errors (user_id); + CREATE INDEX IF NOT EXISTS user_viewed_errors_error_id_idx ON public.user_viewed_errors (error_id); + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'platform') THEN + CREATE TYPE platform AS ENUM ('web','ios','android'); + END IF; + END; + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'device_type') THEN + CREATE TYPE device_type AS ENUM ('desktop','tablet','mobile','other'); + END IF; + END; + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'country') THEN + CREATE TYPE country AS ENUM ('UN','RW','SO','YE','IQ','SA','IR','CY','TZ','SY','AM','KE','CD','DJ','UG','CF','SC','JO','LB','KW','OM','QA','BH','AE','IL','TR','ET','ER','EG','SD','GR','BI','EE','LV','AZ','LT','SJ','GE','MD','BY','FI','AX','UA','MK','HU','BG','AL','PL','RO','XK','ZW','ZM','KM','MW','LS','BW','MU','SZ','RE','ZA','YT','MZ','MG','AF','PK','BD','TM','TJ','LK','BT','IN','MV','IO','NP','MM','UZ','KZ','KG','TF','HM','CC','PW','VN','TH','ID','LA','TW','PH','MY','CN','HK','BN','MO','KH','KR','JP','KP','SG','CK','TL','RU','MN','AU','CX','MH','FM','PG','SB','TV','NR','VU','NC','NF','NZ','FJ','LY','CM','SN','CG','PT','LR','CI','GH','GQ','NG','BF','TG','GW','MR','BJ','GA','SL','ST','GI','GM','GN','TD','NE','ML','EH','TN','ES','MA','MT','DZ','FO','DK','IS','GB','CH','SE','NL','AT','BE','DE','LU','IE','MC','FR','AD','LI','JE','IM','GG','SK','CZ','NO','VA','SM','IT','SI','ME','HR','BA','AO','NA','SH','BV','BB','CV','GY','GF','SR','PM','GL','PY','UY','BR','FK','GS','JM','DO','CU','MQ','BS','BM','AI','TT','KN','DM','AG','LC','TC','AW','VG','VC','MS','MF','BL','GP','GD','KY','BZ','SV','GT','HN','NI','CR','VE','EC','CO','PA','HT','AR','CL','BO','PE','MX','PF','PN','KI','TK','TO','WF','WS','NU','MP','GU','PR','VI','UM','AS','CA','US','PS','RS','AQ','SX','CW','BQ','SS'); + END IF; + END; + CREATE TABLE IF NOT EXISTS sessions ( session_id bigint PRIMARY KEY, project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, @@ -552,157 +626,262 @@ $$ metadata_8 text DEFAULT NULL, metadata_9 text DEFAULT NULL, metadata_10 text DEFAULT NULL --- , --- rehydration_id integer REFERENCES rehydrations(rehydration_id) ON DELETE SET NULL ); - CREATE INDEX sessions_project_id_start_ts_idx ON sessions (project_id, start_ts); - CREATE INDEX sessions_project_id_user_id_idx ON sessions (project_id, user_id); - CREATE INDEX sessions_project_id_user_anonymous_id_idx ON sessions (project_id, user_anonymous_id); - CREATE INDEX sessions_project_id_user_device_idx ON sessions (project_id, user_device); - CREATE INDEX sessions_project_id_user_country_idx ON sessions (project_id, user_country); - CREATE INDEX sessions_project_id_user_browser_idx ON sessions (project_id, user_browser); - CREATE INDEX sessions_project_id_metadata_1_idx ON sessions (project_id, metadata_1); - CREATE INDEX sessions_project_id_metadata_2_idx ON sessions (project_id, metadata_2); - CREATE INDEX sessions_project_id_metadata_3_idx ON sessions (project_id, metadata_3); - CREATE INDEX sessions_project_id_metadata_4_idx ON sessions (project_id, metadata_4); - CREATE INDEX sessions_project_id_metadata_5_idx ON sessions (project_id, metadata_5); - CREATE INDEX sessions_project_id_metadata_6_idx ON sessions (project_id, metadata_6); - CREATE INDEX sessions_project_id_metadata_7_idx ON sessions (project_id, metadata_7); - CREATE INDEX sessions_project_id_metadata_8_idx ON sessions (project_id, metadata_8); - CREATE INDEX sessions_project_id_metadata_9_idx ON sessions (project_id, metadata_9); - CREATE INDEX sessions_project_id_metadata_10_idx ON sessions (project_id, metadata_10); - CREATE INDEX sessions_project_id_watchdogs_score_idx ON sessions (project_id, watchdogs_score DESC); - CREATE INDEX sessions_platform_idx ON public.sessions (platform); + CREATE INDEX IF NOT EXISTS sessions_project_id_start_ts_idx ON sessions (project_id, start_ts); + CREATE INDEX IF NOT EXISTS sessions_project_id_user_id_idx ON sessions (project_id, user_id); + CREATE INDEX IF NOT EXISTS sessions_project_id_user_anonymous_id_idx ON sessions (project_id, user_anonymous_id); + CREATE INDEX IF NOT EXISTS sessions_project_id_user_device_idx ON sessions (project_id, user_device); + CREATE INDEX IF NOT EXISTS sessions_project_id_user_country_idx ON sessions (project_id, user_country); + CREATE INDEX IF NOT EXISTS sessions_project_id_user_browser_idx ON sessions (project_id, user_browser); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_1_idx ON sessions (project_id, metadata_1); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_2_idx ON sessions (project_id, metadata_2); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_3_idx ON sessions (project_id, metadata_3); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_4_idx ON sessions (project_id, metadata_4); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_5_idx ON sessions (project_id, metadata_5); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_6_idx ON sessions (project_id, metadata_6); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_7_idx ON sessions (project_id, metadata_7); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_8_idx ON sessions (project_id, metadata_8); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_9_idx ON sessions (project_id, metadata_9); + CREATE INDEX IF NOT EXISTS sessions_project_id_metadata_10_idx ON sessions (project_id, metadata_10); + CREATE INDEX IF NOT EXISTS sessions_project_id_watchdogs_score_idx ON sessions (project_id, watchdogs_score DESC); + CREATE INDEX IF NOT EXISTS sessions_platform_idx ON public.sessions (platform); - CREATE INDEX sessions_metadata1_gin_idx ON public.sessions USING GIN (metadata_1 gin_trgm_ops); - CREATE INDEX sessions_metadata2_gin_idx ON public.sessions USING GIN (metadata_2 gin_trgm_ops); - CREATE INDEX sessions_metadata3_gin_idx ON public.sessions USING GIN (metadata_3 gin_trgm_ops); - CREATE INDEX sessions_metadata4_gin_idx ON public.sessions USING GIN (metadata_4 gin_trgm_ops); - CREATE INDEX sessions_metadata5_gin_idx ON public.sessions USING GIN (metadata_5 gin_trgm_ops); - CREATE INDEX sessions_metadata6_gin_idx ON public.sessions USING GIN (metadata_6 gin_trgm_ops); - CREATE INDEX sessions_metadata7_gin_idx ON public.sessions USING GIN (metadata_7 gin_trgm_ops); - CREATE INDEX sessions_metadata8_gin_idx ON public.sessions USING GIN (metadata_8 gin_trgm_ops); - CREATE INDEX sessions_metadata9_gin_idx ON public.sessions USING GIN (metadata_9 gin_trgm_ops); - CREATE INDEX sessions_metadata10_gin_idx ON public.sessions USING GIN (metadata_10 gin_trgm_ops); - CREATE INDEX sessions_user_os_gin_idx ON public.sessions USING GIN (user_os gin_trgm_ops); - CREATE INDEX sessions_user_browser_gin_idx ON public.sessions USING GIN (user_browser gin_trgm_ops); - CREATE INDEX sessions_user_device_gin_idx ON public.sessions USING GIN (user_device gin_trgm_ops); - CREATE INDEX sessions_user_id_gin_idx ON public.sessions USING GIN (user_id gin_trgm_ops); - CREATE INDEX sessions_user_anonymous_id_gin_idx ON public.sessions USING GIN (user_anonymous_id gin_trgm_ops); - CREATE INDEX sessions_user_country_gin_idx ON public.sessions (project_id, user_country); - CREATE INDEX sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; - CREATE INDEX sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; - CREATE INDEX sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; - CREATE INDEX sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; - CREATE INDEX sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; - CREATE INDEX sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; - CREATE INDEX sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); - CREATE INDEX sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); - CREATE INDEX sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); - - ALTER TABLE public.sessions - ADD CONSTRAINT web_browser_constraint CHECK ( - (sessions.platform = 'web' AND sessions.user_browser NOTNULL) OR - (sessions.platform != 'web' AND sessions.user_browser ISNULL)); - - ALTER TABLE public.sessions - ADD CONSTRAINT web_user_browser_version_constraint CHECK ( sessions.platform = 'web' OR sessions.user_browser_version ISNULL); - - ALTER TABLE public.sessions - ADD CONSTRAINT web_user_agent_constraint CHECK ( - (sessions.platform = 'web' AND sessions.user_agent NOTNULL) OR - (sessions.platform != 'web' AND sessions.user_agent ISNULL)); - - - CREATE TABLE user_viewed_sessions + CREATE INDEX IF NOT EXISTS sessions_metadata1_gin_idx ON public.sessions USING GIN (metadata_1 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata2_gin_idx ON public.sessions USING GIN (metadata_2 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata3_gin_idx ON public.sessions USING GIN (metadata_3 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata4_gin_idx ON public.sessions USING GIN (metadata_4 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata5_gin_idx ON public.sessions USING GIN (metadata_5 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata6_gin_idx ON public.sessions USING GIN (metadata_6 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata7_gin_idx ON public.sessions USING GIN (metadata_7 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata8_gin_idx ON public.sessions USING GIN (metadata_8 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata9_gin_idx ON public.sessions USING GIN (metadata_9 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_metadata10_gin_idx ON public.sessions USING GIN (metadata_10 gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_user_os_gin_idx ON public.sessions USING GIN (user_os gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_user_browser_gin_idx ON public.sessions USING GIN (user_browser gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_user_device_gin_idx ON public.sessions USING GIN (user_device gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_user_id_gin_idx ON public.sessions USING GIN (user_id gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_user_anonymous_id_gin_idx ON public.sessions USING GIN (user_anonymous_id gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_user_country_gin_idx ON public.sessions (project_id, user_country); + CREATE INDEX IF NOT EXISTS sessions_start_ts_idx ON public.sessions (start_ts) WHERE duration > 0; + CREATE INDEX IF NOT EXISTS sessions_project_id_idx ON public.sessions (project_id) WHERE duration > 0; + CREATE INDEX IF NOT EXISTS sessions_session_id_project_id_start_ts_idx ON sessions (session_id, project_id, start_ts) WHERE duration > 0; + CREATE INDEX IF NOT EXISTS sessions_session_id_project_id_start_ts_durationNN_idx ON sessions (session_id, project_id, start_ts) WHERE duration IS NOT NULL; + CREATE INDEX IF NOT EXISTS sessions_user_id_useridNN_idx ON sessions (user_id) WHERE user_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS sessions_uid_projectid_startts_sessionid_uidNN_durGTZ_idx ON sessions (user_id, project_id, start_ts, session_id) WHERE user_id IS NOT NULL AND duration > 0; + CREATE INDEX IF NOT EXISTS sessions_utm_source_gin_idx ON public.sessions USING GIN (utm_source gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_utm_medium_gin_idx ON public.sessions USING GIN (utm_medium gin_trgm_ops); + CREATE INDEX IF NOT EXISTS sessions_utm_campaign_gin_idx ON public.sessions USING GIN (utm_campaign gin_trgm_ops); + BEGIN + ALTER TABLE public.sessions + ADD CONSTRAINT web_browser_constraint CHECK ( + (sessions.platform = 'web' AND sessions.user_browser NOTNULL) OR + (sessions.platform != 'web' AND sessions.user_browser ISNULL)); + EXCEPTION + WHEN duplicate_object THEN RAISE NOTICE 'Table constraint exists'; + END; + BEGIN + ALTER TABLE public.sessions + ADD CONSTRAINT web_user_browser_version_constraint CHECK ( + sessions.platform = 'web' OR sessions.user_browser_version ISNULL); + EXCEPTION + WHEN duplicate_object THEN RAISE NOTICE 'Table constraint exists'; + END; + BEGIN + ALTER TABLE public.sessions + ADD CONSTRAINT web_user_agent_constraint CHECK ( + (sessions.platform = 'web' AND sessions.user_agent NOTNULL) OR + (sessions.platform != 'web' AND sessions.user_agent ISNULL)); + EXCEPTION + WHEN duplicate_object THEN RAISE NOTICE 'Table constraint already exists'; + END; + CREATE TABLE IF NOT EXISTS user_viewed_sessions ( user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, PRIMARY KEY (user_id, session_id) ); - CREATE TABLE user_favorite_sessions + CREATE TABLE IF NOT EXISTS user_favorite_sessions ( user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, PRIMARY KEY (user_id, session_id) ); - CREATE INDEX user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); + CREATE INDEX IF NOT EXISTS user_favorite_sessions_user_id_session_id_idx ON user_favorite_sessions (user_id, session_id); --- --- assignments.sql --- - create table assigned_sessions + CREATE TABLE IF NOT EXISTS assigned_sessions ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, issue_id text NOT NULL, provider oauth_provider NOT NULL, created_by integer NOT NULL, created_at timestamp default timezone('utc'::text, now()) NOT NULL, - provider_data jsonb default '{}'::jsonb NOT NULL + provider_data jsonb default'{}'::jsonb NOT NULL ); - CREATE INDEX assigned_sessions_session_id_idx ON assigned_sessions (session_id); + CREATE INDEX IF NOT EXISTS assigned_sessions_session_id_idx ON assigned_sessions (session_id); --- --- events_common.sql --- - CREATE SCHEMA IF NOT EXISTS events_common; - - CREATE TYPE events_common.custom_level AS ENUM ('info','error'); - - CREATE TABLE events_common.customs + CREATE TABLE IF NOT EXISTS autocomplete ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - name text NOT NULL, - payload jsonb NOT NULL, - level events_common.custom_level NOT NULL DEFAULT 'info', - PRIMARY KEY (session_id, timestamp, seq_index) + value text NOT NULL, + type text NOT NULL, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE ); - CREATE INDEX customs_name_idx ON events_common.customs (name); - CREATE INDEX customs_name_gin_idx ON events_common.customs USING GIN (name gin_trgm_ops); - CREATE INDEX customs_timestamp_idx ON events_common.customs (timestamp); + CREATE unique index IF NOT EXISTS autocomplete_unique ON autocomplete (project_id, value, type); + CREATE index IF NOT EXISTS autocomplete_project_id_idx ON autocomplete (project_id); + CREATE INDEX IF NOT EXISTS autocomplete_type_idx ON public.autocomplete (type); + CREATE INDEX IF NOT EXISTS autocomplete_value_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops); - CREATE TABLE events_common.issues + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'job_status') THEN + CREATE TYPE job_status AS ENUM ('scheduled','running','cancelled','failed','completed'); + END IF; + END; + BEGIN + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'job_action') THEN + CREATE TYPE job_action AS ENUM ('delete_user_data'); + END IF; + END; + CREATE TABLE IF NOT EXISTS jobs ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - issue_id text NOT NULL REFERENCES issues (issue_id) ON DELETE CASCADE, - payload jsonb DEFAULT NULL, - PRIMARY KEY (session_id, timestamp, seq_index) + job_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + description text NOT NULL, + status job_status NOT NULL, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + action job_action NOT NULL, + reference_id text NOT NULL, + created_at timestamp default timezone('utc'::text, now()) NOT NULL, + updated_at timestamp default timezone('utc'::text, now()) NULL, + start_at timestamp NOT NULL, + errors text NULL ); - CREATE INDEX issues_issue_id_timestamp_idx ON events_common.issues (issue_id, timestamp); - CREATE INDEX issues_timestamp_idx ON events_common.issues (timestamp); + CREATE INDEX IF NOT EXISTS jobs_status_idx ON jobs (status); + CREATE INDEX IF NOT EXISTS jobs_start_at_idx ON jobs (start_at); + CREATE INDEX IF NOT EXISTS jobs_project_id_idx ON jobs (project_id); - CREATE TABLE events_common.requests + CREATE TABLE IF NOT EXISTS traces ( - session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, - timestamp bigint NOT NULL, - seq_index integer NOT NULL, - url text NOT NULL, - duration integer NOT NULL, - success boolean NOT NULL, - PRIMARY KEY (session_id, timestamp, seq_index) + user_id integer NULL REFERENCES users (user_id) ON DELETE CASCADE, + tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, + created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, + auth text NULL, + action text NOT NULL, + method text NOT NULL, + path_format text NOT NULL, + endpoint text NOT NULL, + payload jsonb NULL, + parameters jsonb NULL, + status int NULL ); - CREATE INDEX requests_url_idx ON events_common.requests (url); - CREATE INDEX requests_duration_idx ON events_common.requests (duration); - CREATE INDEX requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); - CREATE INDEX requests_timestamp_idx ON events_common.requests (timestamp); - CREATE INDEX requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, length(url) - (CASE - WHEN url LIKE 'http://%' - THEN 7 - WHEN url LIKE 'https://%' - THEN 8 - ELSE 0 END)) - gin_trgm_ops); - CREATE INDEX requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + CREATE INDEX IF NOT EXISTS traces_user_id_idx ON traces (user_id); + CREATE INDEX IF NOT EXISTS traces_tenant_id_idx ON traces (tenant_id); --- --- events.sql --- - CREATE SCHEMA IF NOT EXISTS events; + CREATE TABLE IF NOT EXISTS metrics + ( + metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer REFERENCES users (user_id) ON DELETE SET NULL, + name text NOT NULL, + is_public boolean NOT NULL DEFAULT FALSE, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp + ); + CREATE INDEX IF NOT EXISTS metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); + CREATE TABLE IF NOT EXISTS metric_series + ( + series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, + index integer NOT NULL, + name text NULL, + filter jsonb NOT NULL, + created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, + deleted_at timestamp + ); + CREATE INDEX IF NOT EXISTS metric_series_metric_id_idx ON public.metric_series (metric_id); - CREATE TABLE events.pages + CREATE TABLE IF NOT EXISTS searches + ( + search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + name text not null, + filter jsonb not null, + created_at timestamp default timezone('utc'::text, now()) not null, + deleted_at timestamp, + is_public boolean NOT NULL DEFAULT False + ); + + CREATE INDEX IF NOT EXISTS searches_user_id_is_public_idx ON public.searches (user_id, is_public); + CREATE INDEX IF NOT EXISTS searches_project_id_idx ON public.searches (project_id); + + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'alert_detection_method') THEN + CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); + END IF; + CREATE TABLE IF NOT EXISTS alerts + ( + alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, + project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, + series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE, + name text NOT NULL, + description text NULL DEFAULT NULL, + active boolean NOT NULL DEFAULT TRUE, + detection_method alert_detection_method NOT NULL, + query jsonb NOT NULL, + deleted_at timestamp NULL DEFAULT NULL, + created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), + options jsonb NOT NULL DEFAULT'{ + "renotifyInterval": 1440 + }'::jsonb + ); + CREATE INDEX IF NOT EXISTS alerts_project_id_idx ON alerts (project_id); + CREATE INDEX IF NOT EXISTS alerts_series_id_idx ON alerts (series_id); + + DROP TRIGGER IF EXISTS on_insert_or_update_or_delete ON alerts; + + CREATE TRIGGER on_insert_or_update_or_delete + AFTER INSERT OR UPDATE OR DELETE + ON alerts + FOR EACH ROW + EXECUTE PROCEDURE notify_alert(); + + RAISE NOTICE 'Created missing public schema tables'; + END IF; + END; + +$$ +LANGUAGE plpgsql; + + +DO +$$ + BEGIN + IF (with to_check (name) as ( + values ('clicks'), + ('errors'), + ('graphql'), + ('inputs'), + ('pages'), + ('performance'), + ('resources'), + ('state_actions') + ) + select bool_and(exists(select * + from information_schema.tables t + where table_schema = 'events' + AND table_name = to_check.name)) as all_present + from to_check) THEN + raise notice 'All events schema tables exists'; + ELSE + CREATE TABLE IF NOT EXISTS events.pages ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -725,48 +904,49 @@ $$ ttfb integer DEFAULT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX pages_session_id_idx ON events.pages (session_id); - CREATE INDEX pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); - CREATE INDEX pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); - CREATE INDEX pages_timestamp_idx ON events.pages (timestamp); - CREATE INDEX pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); - CREATE INDEX pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); - CREATE INDEX pages_base_path_idx ON events.pages (base_path); - CREATE INDEX pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); - CREATE INDEX pages_base_referrer_idx ON events.pages (base_referrer); - CREATE INDEX pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, - length(base_referrer) - (CASE - WHEN base_referrer LIKE 'http://%' - THEN 7 - WHEN base_referrer LIKE 'https://%' - THEN 8 - ELSE 0 END)) - gin_trgm_ops); - CREATE INDEX pages_response_time_idx ON events.pages (response_time); - CREATE INDEX pages_response_end_idx ON events.pages (response_end); - CREATE INDEX pages_path_gin_idx ON events.pages USING GIN (path gin_trgm_ops); - CREATE INDEX pages_path_idx ON events.pages (path); - CREATE INDEX pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; - CREATE INDEX pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0; - CREATE INDEX pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0; - CREATE INDEX pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; - CREATE INDEX pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; - CREATE INDEX pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; - CREATE INDEX pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; - CREATE INDEX pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; - CREATE INDEX pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; - CREATE INDEX pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; - CREATE INDEX pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR - first_paint_time > 0 OR - dom_content_loaded_time > 0 OR - ttfb > 0 OR - time_to_interactive > 0; - CREATE INDEX pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; - CREATE INDEX pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; - CREATE INDEX pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); - CREATE INDEX pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; + CREATE INDEX IF NOT EXISTS pages_session_id_idx ON events.pages (session_id); + CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx ON events.pages USING GIN (base_path gin_trgm_ops); + CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx ON events.pages USING GIN (base_referrer gin_trgm_ops); + CREATE INDEX IF NOT EXISTS pages_timestamp_idx ON events.pages (timestamp); + CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_idx ON events.pages (session_id, timestamp); + CREATE INDEX IF NOT EXISTS pages_base_path_gin_idx2 ON events.pages USING GIN (RIGHT(base_path, length(base_path) - 1) gin_trgm_ops); + CREATE INDEX IF NOT EXISTS pages_base_path_idx ON events.pages (base_path); + CREATE INDEX IF NOT EXISTS pages_base_path_idx2 ON events.pages (RIGHT(base_path, length(base_path) - 1)); + CREATE INDEX IF NOT EXISTS pages_base_referrer_idx ON events.pages (base_referrer); + CREATE INDEX IF NOT EXISTS pages_base_referrer_gin_idx2 ON events.pages USING GIN (RIGHT(base_referrer, + length(base_referrer) - + (CASE + WHEN base_referrer LIKE 'http://%' + THEN 7 + WHEN base_referrer LIKE 'https://%' + THEN 8 + ELSE 0 END)) + gin_trgm_ops); + CREATE INDEX IF NOT EXISTS pages_response_time_idx ON events.pages (response_time); + CREATE INDEX IF NOT EXISTS pages_response_end_idx ON events.pages (response_end); + CREATE INDEX IF NOT EXISTS pages_path_gin_idx ON events.pages USING GIN (path gin_trgm_ops); + CREATE INDEX IF NOT EXISTS pages_path_idx ON events.pages (path); + CREATE INDEX IF NOT EXISTS pages_visually_complete_idx ON events.pages (visually_complete) WHERE visually_complete > 0; + CREATE INDEX IF NOT EXISTS pages_dom_building_time_idx ON events.pages (dom_building_time) WHERE dom_building_time > 0; + CREATE INDEX IF NOT EXISTS pages_load_time_idx ON events.pages (load_time) WHERE load_time > 0; + CREATE INDEX IF NOT EXISTS pages_first_contentful_paint_time_idx ON events.pages (first_contentful_paint_time) WHERE first_contentful_paint_time > 0; + CREATE INDEX IF NOT EXISTS pages_dom_content_loaded_time_idx ON events.pages (dom_content_loaded_time) WHERE dom_content_loaded_time > 0; + CREATE INDEX IF NOT EXISTS pages_first_paint_time_idx ON events.pages (first_paint_time) WHERE first_paint_time > 0; + CREATE INDEX IF NOT EXISTS pages_ttfb_idx ON events.pages (ttfb) WHERE ttfb > 0; + CREATE INDEX IF NOT EXISTS pages_time_to_interactive_idx ON events.pages (time_to_interactive) WHERE time_to_interactive > 0; + CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_loadgt0NN_idx ON events.pages (session_id, timestamp) WHERE load_time > 0 AND load_time IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_visualgt0nn_idx ON events.pages (session_id, timestamp) WHERE visually_complete > 0 AND visually_complete IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_timestamp_metgt0_idx ON events.pages (timestamp) WHERE response_time > 0 OR + first_paint_time > 0 OR + dom_content_loaded_time > 0 OR + ttfb > 0 OR + time_to_interactive > 0; + CREATE INDEX IF NOT EXISTS pages_session_id_speed_indexgt0nn_idx ON events.pages (session_id, speed_index) WHERE speed_index > 0 AND speed_index IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_session_id_timestamp_dom_building_timegt0nn_idx ON events.pages (session_id, timestamp, dom_building_time) WHERE dom_building_time > 0 AND dom_building_time IS NOT NULL; + CREATE INDEX IF NOT EXISTS pages_base_path_session_id_timestamp_idx ON events.pages (base_path, session_id, timestamp); + CREATE INDEX IF NOT EXISTS pages_base_path_base_pathLNGT2_idx ON events.pages (base_path) WHERE length(base_path) > 2; - CREATE TABLE events.clicks + CREATE TABLE IF NOT EXISTS events.clicks ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -776,18 +956,18 @@ $$ selector text DEFAULT '' NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX clicks_session_id_idx ON events.clicks (session_id); - CREATE INDEX clicks_label_idx ON events.clicks (label); - CREATE INDEX clicks_label_gin_idx ON events.clicks USING GIN (label gin_trgm_ops); - CREATE INDEX clicks_timestamp_idx ON events.clicks (timestamp); - CREATE INDEX clicks_label_session_id_timestamp_idx ON events.clicks (label, session_id, timestamp); - CREATE INDEX clicks_url_idx ON events.clicks (url); - CREATE INDEX clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops); - CREATE INDEX clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector); - CREATE INDEX clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); + CREATE INDEX IF NOT EXISTS clicks_session_id_idx ON events.clicks (session_id); + CREATE INDEX IF NOT EXISTS clicks_label_idx ON events.clicks (label); + CREATE INDEX IF NOT EXISTS clicks_label_gin_idx ON events.clicks USING GIN (label gin_trgm_ops); + CREATE INDEX IF NOT EXISTS clicks_timestamp_idx ON events.clicks (timestamp); + CREATE INDEX IF NOT EXISTS clicks_label_session_id_timestamp_idx ON events.clicks (label, session_id, timestamp); + CREATE INDEX IF NOT EXISTS clicks_url_idx ON events.clicks (url); + CREATE INDEX IF NOT EXISTS clicks_url_gin_idx ON events.clicks USING GIN (url gin_trgm_ops); + CREATE INDEX IF NOT EXISTS clicks_url_session_id_timestamp_selector_idx ON events.clicks (url, session_id, timestamp, selector); + CREATE INDEX IF NOT EXISTS clicks_session_id_timestamp_idx ON events.clicks (session_id, timestamp); - CREATE TABLE events.inputs + CREATE TABLE IF NOT EXISTS events.inputs ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -796,14 +976,14 @@ $$ value text DEFAULT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX inputs_session_id_idx ON events.inputs (session_id); - CREATE INDEX inputs_label_value_idx ON events.inputs (label, value); - CREATE INDEX inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); - CREATE INDEX inputs_label_idx ON events.inputs (label); - CREATE INDEX inputs_timestamp_idx ON events.inputs (timestamp); - CREATE INDEX inputs_label_session_id_timestamp_idx ON events.inputs (label, session_id, timestamp); + CREATE INDEX IF NOT EXISTS inputs_session_id_idx ON events.inputs (session_id); + CREATE INDEX IF NOT EXISTS inputs_label_value_idx ON events.inputs (label, value); + CREATE INDEX IF NOT EXISTS inputs_label_gin_idx ON events.inputs USING GIN (label gin_trgm_ops); + CREATE INDEX IF NOT EXISTS inputs_label_idx ON events.inputs (label); + CREATE INDEX IF NOT EXISTS inputs_timestamp_idx ON events.inputs (timestamp); + CREATE INDEX IF NOT EXISTS inputs_label_session_id_timestamp_idx ON events.inputs (label, session_id, timestamp); - CREATE TABLE events.errors + CREATE TABLE IF NOT EXISTS events.errors ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -811,16 +991,16 @@ $$ error_id text NOT NULL REFERENCES errors (error_id) ON DELETE CASCADE, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX errors_session_id_idx ON events.errors (session_id); - CREATE INDEX errors_timestamp_idx ON events.errors (timestamp); - CREATE INDEX errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); - CREATE INDEX errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); - CREATE INDEX errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); - CREATE INDEX errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); - CREATE INDEX errors_error_id_idx ON events.errors (error_id); + CREATE INDEX IF NOT EXISTS errors_session_id_idx ON events.errors (session_id); + CREATE INDEX IF NOT EXISTS errors_timestamp_idx ON events.errors (timestamp); + CREATE INDEX IF NOT EXISTS errors_session_id_timestamp_error_id_idx ON events.errors (session_id, timestamp, error_id); + CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_idx ON events.errors (error_id, timestamp); + CREATE INDEX IF NOT EXISTS errors_timestamp_error_id_session_id_idx ON events.errors (timestamp, error_id, session_id); + CREATE INDEX IF NOT EXISTS errors_error_id_timestamp_session_id_idx ON events.errors (error_id, timestamp, session_id); + CREATE INDEX IF NOT EXISTS errors_error_id_idx ON events.errors (error_id); - CREATE TABLE events.graphql + CREATE TABLE IF NOT EXISTS events.graphql ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -828,11 +1008,11 @@ $$ name text NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX graphql_name_idx ON events.graphql (name); - CREATE INDEX graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); - CREATE INDEX graphql_timestamp_idx ON events.graphql (timestamp); + CREATE INDEX IF NOT EXISTS graphql_name_idx ON events.graphql (name); + CREATE INDEX IF NOT EXISTS graphql_name_gin_idx ON events.graphql USING GIN (name gin_trgm_ops); + CREATE INDEX IF NOT EXISTS graphql_timestamp_idx ON events.graphql (timestamp); - CREATE TABLE events.state_actions + CREATE TABLE IF NOT EXISTS events.state_actions ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -840,13 +1020,21 @@ $$ name text NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX state_actions_name_idx ON events.state_actions (name); - CREATE INDEX state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); - CREATE INDEX state_actions_timestamp_idx ON events.state_actions (timestamp); + CREATE INDEX IF NOT EXISTS state_actions_name_idx ON events.state_actions (name); + CREATE INDEX IF NOT EXISTS state_actions_name_gin_idx ON events.state_actions USING GIN (name gin_trgm_ops); + CREATE INDEX IF NOT EXISTS state_actions_timestamp_idx ON events.state_actions (timestamp); - CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); - CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); - CREATE TABLE events.resources + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'resource_type') THEN + CREATE TYPE events.resource_type AS ENUM ('other', 'script', 'stylesheet', 'fetch', 'img', 'media'); + END IF; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'resource_method') THEN + CREATE TYPE events.resource_method AS ENUM ('GET' , 'HEAD' , 'POST' , 'PUT' , 'DELETE' , 'CONNECT' , 'OPTIONS' , 'TRACE' , 'PATCH' ); + END IF; + CREATE TABLE IF NOT EXISTS events.resources ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, message_id bigint NOT NULL, @@ -865,28 +1053,28 @@ $$ decoded_body_size integer NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX resources_session_id_idx ON events.resources (session_id); - CREATE INDEX resources_status_idx ON events.resources (status); - CREATE INDEX resources_type_idx ON events.resources (type); - CREATE INDEX resources_duration_durationgt0_idx ON events.resources (duration) WHERE duration > 0; - CREATE INDEX resources_url_host_idx ON events.resources (url_host); - CREATE INDEX resources_timestamp_idx ON events.resources (timestamp); - CREATE INDEX resources_success_idx ON events.resources (success); + CREATE INDEX IF NOT EXISTS resources_session_id_idx ON events.resources (session_id); + CREATE INDEX IF NOT EXISTS resources_status_idx ON events.resources (status); + CREATE INDEX IF NOT EXISTS resources_type_idx ON events.resources (type); + CREATE INDEX IF NOT EXISTS resources_duration_durationgt0_idx ON events.resources (duration) WHERE duration > 0; + CREATE INDEX IF NOT EXISTS resources_url_host_idx ON events.resources (url_host); + CREATE INDEX IF NOT EXISTS resources_timestamp_idx ON events.resources (timestamp); + CREATE INDEX IF NOT EXISTS resources_success_idx ON events.resources (success); - CREATE INDEX resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); - CREATE INDEX resources_url_idx ON events.resources (url); - CREATE INDEX resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); - CREATE INDEX resources_url_hostpath_idx ON events.resources (url_hostpath); - CREATE INDEX resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; - CREATE INDEX resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); - CREATE INDEX resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); - CREATE INDEX resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; - CREATE INDEX resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; - CREATE INDEX resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); - CREATE INDEX resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; - CREATE INDEX resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); + CREATE INDEX IF NOT EXISTS resources_url_gin_idx ON events.resources USING GIN (url gin_trgm_ops); + CREATE INDEX IF NOT EXISTS resources_url_idx ON events.resources (url); + CREATE INDEX IF NOT EXISTS resources_url_hostpath_gin_idx ON events.resources USING GIN (url_hostpath gin_trgm_ops); + CREATE INDEX IF NOT EXISTS resources_url_hostpath_idx ON events.resources (url_hostpath); + CREATE INDEX IF NOT EXISTS resources_timestamp_type_durationgt0NN_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL; + CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_idx ON events.resources (session_id, timestamp); + CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_type_idx ON events.resources (session_id, timestamp, type); + CREATE INDEX IF NOT EXISTS resources_timestamp_type_durationgt0NN_noFetch_idx ON events.resources (timestamp, type) WHERE duration > 0 AND duration IS NOT NULL AND type != 'fetch'; + CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_url_host_fail_idx ON events.resources (session_id, timestamp, url_host) WHERE success = FALSE; + CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_url_host_firstparty_idx ON events.resources (session_id, timestamp, url_host) WHERE type IN ('fetch', 'script'); + CREATE INDEX IF NOT EXISTS resources_session_id_timestamp_duration_durationgt0NN_img_idx ON events.resources (session_id, timestamp, duration) WHERE duration > 0 AND duration IS NOT NULL AND type = 'img'; + CREATE INDEX IF NOT EXISTS resources_timestamp_session_id_idx ON events.resources (timestamp, session_id); - CREATE TABLE events.performance + CREATE TABLE IF NOT EXISTS events.performance ( session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, timestamp bigint NOT NULL, @@ -905,132 +1093,93 @@ $$ max_used_js_heap_size bigint NOT NULL, PRIMARY KEY (session_id, message_id) ); - CREATE INDEX performance_session_id_idx ON events.performance (session_id); - CREATE INDEX performance_timestamp_idx ON events.performance (timestamp); - CREATE INDEX performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); - CREATE INDEX performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; - CREATE INDEX performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; - --- --- autocomplete.sql --- - - CREATE TABLE autocomplete - ( - value text NOT NULL, - type text NOT NULL, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE - ); - - CREATE unique index autocomplete_unique ON autocomplete (project_id, value, type); - CREATE index autocomplete_project_id_idx ON autocomplete (project_id); - CREATE INDEX autocomplete_type_idx ON public.autocomplete (type); - CREATE INDEX autocomplete_value_gin_idx ON public.autocomplete USING GIN (value gin_trgm_ops); - --- --- jobs.sql --- - CREATE TYPE job_status AS ENUM ('scheduled','running','cancelled','failed','completed'); - CREATE TYPE job_action AS ENUM ('delete_user_data'); - CREATE TABLE jobs - ( - job_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - description text NOT NULL, - status job_status NOT NULL, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - action job_action NOT NULL, - reference_id text NOT NULL, - created_at timestamp default timezone('utc'::text, now()) NOT NULL, - updated_at timestamp default timezone('utc'::text, now()) NULL, - start_at timestamp NOT NULL, - errors text NULL - ); - CREATE INDEX jobs_status_idx ON jobs (status); - CREATE INDEX jobs_start_at_idx ON jobs (start_at); - CREATE INDEX jobs_project_id_idx ON jobs (project_id); - - - CREATE TABLE traces - ( - user_id integer NULL REFERENCES users (user_id) ON DELETE CASCADE, - tenant_id integer NOT NULL REFERENCES tenants (tenant_id) ON DELETE CASCADE, - created_at bigint NOT NULL DEFAULT (EXTRACT(EPOCH FROM now() at time zone 'utc') * 1000)::bigint, - auth text NULL, - action text NOT NULL, - method text NOT NULL, - path_format text NOT NULL, - endpoint text NOT NULL, - payload jsonb NULL, - parameters jsonb NULL, - status int NULL - ); - CREATE INDEX traces_user_id_idx ON traces (user_id); - CREATE INDEX traces_tenant_id_idx ON traces (tenant_id); - - CREATE TABLE metrics - ( - metric_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer REFERENCES users (user_id) ON DELETE SET NULL, - name text NOT NULL, - is_public boolean NOT NULL DEFAULT FALSE, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp - ); - CREATE INDEX metrics_user_id_is_public_idx ON public.metrics (user_id, is_public); - CREATE TABLE metric_series - ( - series_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - metric_id integer REFERENCES metrics (metric_id) ON DELETE CASCADE, - index integer NOT NULL, - name text NULL, - filter jsonb NOT NULL, - created_at timestamp DEFAULT timezone('utc'::text, now()) NOT NULL, - deleted_at timestamp - ); - CREATE INDEX metric_series_metric_id_idx ON public.metric_series (metric_id); - - CREATE TABLE searches - ( - search_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - user_id integer NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, - name text not null, - filter jsonb not null, - created_at timestamp default timezone('utc'::text, now()) not null, - deleted_at timestamp, - is_public boolean NOT NULL DEFAULT False - ); - - CREATE INDEX searches_user_id_is_public_idx ON public.searches (user_id, is_public); - CREATE INDEX searches_project_id_idx ON public.searches (project_id); - - CREATE TYPE alert_detection_method AS ENUM ('threshold', 'change'); - - CREATE TABLE alerts - ( - alert_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, - project_id integer NOT NULL REFERENCES projects (project_id) ON DELETE CASCADE, - series_id integer NULL REFERENCES metric_series (series_id) ON DELETE CASCADE, - name text NOT NULL, - description text NULL DEFAULT NULL, - active boolean NOT NULL DEFAULT TRUE, - detection_method alert_detection_method NOT NULL, - query jsonb NOT NULL, - deleted_at timestamp NULL DEFAULT NULL, - created_at timestamp NOT NULL DEFAULT timezone('utc'::text, now()), - options jsonb NOT NULL DEFAULT '{ - "renotifyInterval": 1440 - }'::jsonb - ); - CREATE INDEX alerts_project_id_idx ON alerts (project_id); - CREATE INDEX alerts_series_id_idx ON alerts (series_id); - CREATE TRIGGER on_insert_or_update_or_delete - AFTER INSERT OR UPDATE OR DELETE - ON alerts - FOR EACH ROW - EXECUTE PROCEDURE notify_alert(); - - raise notice 'DB created'; + CREATE INDEX IF NOT EXISTS performance_session_id_idx ON events.performance (session_id); + CREATE INDEX IF NOT EXISTS performance_timestamp_idx ON events.performance (timestamp); + CREATE INDEX IF NOT EXISTS performance_session_id_timestamp_idx ON events.performance (session_id, timestamp); + CREATE INDEX IF NOT EXISTS performance_avg_cpu_gt0_idx ON events.performance (avg_cpu) WHERE avg_cpu > 0; + CREATE INDEX IF NOT EXISTS performance_avg_used_js_heap_size_gt0_idx ON events.performance (avg_used_js_heap_size) WHERE avg_used_js_heap_size > 0; END IF; END; - $$ LANGUAGE plpgsql; + + +DO +$$ + BEGIN + IF (with to_check (name) as ( + values ('customs'), + ('issues'), + ('requests') + ) + select bool_and(exists(select * + from information_schema.tables t + where table_schema = 'events_common' + AND table_name = to_check.name)) as all_present + from to_check) THEN + raise notice 'All events_common schema tables exists'; + ELSE + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'custom_level') THEN + CREATE TYPE events_common.custom_level AS ENUM ('info','error'); + END IF; + CREATE TABLE IF NOT EXISTS events_common.customs + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + name text NOT NULL, + payload jsonb NOT NULL, + level events_common.custom_level NOT NULL DEFAULT 'info', + PRIMARY KEY (session_id, timestamp, seq_index) + ); + CREATE INDEX IF NOT EXISTS customs_name_idx ON events_common.customs (name); + CREATE INDEX IF NOT EXISTS customs_name_gin_idx ON events_common.customs USING GIN (name gin_trgm_ops); + CREATE INDEX IF NOT EXISTS customs_timestamp_idx ON events_common.customs (timestamp); + + + CREATE TABLE IF NOT EXISTS events_common.issues + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + issue_id text NOT NULL REFERENCES issues (issue_id) ON DELETE CASCADE, + payload jsonb DEFAULT NULL, + PRIMARY KEY (session_id, timestamp, seq_index) + ); + CREATE INDEX IF NOT EXISTS issues_issue_id_timestamp_idx ON events_common.issues (issue_id, timestamp); + CREATE INDEX IF NOT EXISTS issues_timestamp_idx ON events_common.issues (timestamp); + + + CREATE TABLE IF NOT EXISTS events_common.requests + ( + session_id bigint NOT NULL REFERENCES sessions (session_id) ON DELETE CASCADE, + timestamp bigint NOT NULL, + seq_index integer NOT NULL, + url text NOT NULL, + duration integer NOT NULL, + success boolean NOT NULL, + PRIMARY KEY (session_id, timestamp, seq_index) + ); + CREATE INDEX IF NOT EXISTS requests_url_idx ON events_common.requests (url); + CREATE INDEX IF NOT EXISTS requests_duration_idx ON events_common.requests (duration); + CREATE INDEX IF NOT EXISTS requests_url_gin_idx ON events_common.requests USING GIN (url gin_trgm_ops); + CREATE INDEX IF NOT EXISTS requests_timestamp_idx ON events_common.requests (timestamp); + CREATE INDEX IF NOT EXISTS requests_url_gin_idx2 ON events_common.requests USING GIN (RIGHT(url, + length(url) - + (CASE + WHEN url LIKE 'http://%' + THEN 7 + WHEN url LIKE 'https://%' + THEN 8 + ELSE 0 END)) + gin_trgm_ops); + CREATE INDEX IF NOT EXISTS requests_timestamp_session_id_failed_idx ON events_common.requests (timestamp, session_id) WHERE success = FALSE; + END IF; + END; +$$ +LANGUAGE plpgsql; + + COMMIT; \ No newline at end of file From bf279945d7cc2466d7db36a8a2fd005c4fcf59e3 Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 26 Jan 2022 11:53:27 +0100 Subject: [PATCH 202/206] feat(DB): EE init_schema check for all tables refactored --- .../db/init_dbs/postgresql/init_schema.sql | 169 ++++++++---------- 1 file changed, 78 insertions(+), 91 deletions(-) diff --git a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql index 7db4b0bed..256589c91 100644 --- a/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql +++ b/ee/scripts/helm/db/init_dbs/postgresql/init_schema.sql @@ -1,5 +1,4 @@ BEGIN; - CREATE SCHEMA IF NOT EXISTS events_common; CREATE SCHEMA IF NOT EXISTS events; CREATE EXTENSION IF NOT EXISTS pg_trgm; @@ -337,13 +336,11 @@ $$ CREATE INDEX IF NOT EXISTS roles_projects_project_id_idx ON roles_projects (project_id); - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'webhook_type') THEN - create type webhook_type as enum ('webhook','slack','email'); - END IF; - END; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'webhook_type') THEN + create type webhook_type as enum ('webhook','slack','email'); + END IF; create table IF NOT EXISTS webhooks @@ -408,13 +405,11 @@ $$ CREATE INDEX IF NOT EXISTS funnels_user_id_is_public_idx ON public.funnels (user_id, is_public); CREATE INDEX IF NOT EXISTS funnels_project_id_idx ON public.funnels (project_id); - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'announcement_type') THEN - create type announcement_type as enum ('notification','alert'); - END IF; - END; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'announcement_type') THEN + create type announcement_type as enum ('notification','alert'); + END IF; create table IF NOT EXISTS announcements ( @@ -430,13 +425,11 @@ $$ type announcement_type default 'notification'::announcement_type not null ); - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'integration_provider') THEN - CREATE TYPE integration_provider AS ENUM ('bugsnag','cloudwatch','datadog','newrelic','rollbar','sentry','stackdriver','sumologic','elasticsearch'); --,'jira','github'); - END IF; - END; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'integration_provider') THEN + CREATE TYPE integration_provider AS ENUM ('bugsnag','cloudwatch','datadog','newrelic','rollbar','sentry','stackdriver','sumologic','elasticsearch'); --,'jira','github'); + END IF; CREATE TABLE IF NOT EXISTS integrations ( @@ -469,33 +462,31 @@ $$ url text ); - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'issue_type') THEN - CREATE TYPE issue_type AS ENUM ( - 'click_rage', - 'dead_click', - 'excessive_scrolling', - 'bad_request', - 'missing_resource', - 'memory', - 'cpu', - 'slow_resource', - 'slow_page_load', - 'crash', - 'ml_cpu', - 'ml_memory', - 'ml_dead_click', - 'ml_click_rage', - 'ml_mouse_thrashing', - 'ml_excessive_scrolling', - 'ml_slow_resources', - 'custom', - 'js_exception' - ); - END IF; - END; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'issue_type') THEN + CREATE TYPE issue_type AS ENUM ( + 'click_rage', + 'dead_click', + 'excessive_scrolling', + 'bad_request', + 'missing_resource', + 'memory', + 'cpu', + 'slow_resource', + 'slow_page_load', + 'crash', + 'ml_cpu', + 'ml_memory', + 'ml_dead_click', + 'ml_click_rage', + 'ml_mouse_thrashing', + 'ml_excessive_scrolling', + 'ml_slow_resources', + 'custom', + 'js_exception' + ); + END IF; CREATE TABLE IF NOT EXISTS issues ( @@ -510,20 +501,18 @@ $$ CREATE INDEX IF NOT EXISTS issues_project_id_issue_id_idx ON public.issues (project_id, issue_id); CREATE INDEX IF NOT EXISTS issues_project_id_idx ON issues (project_id); - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'error_source') THEN - CREATE TYPE error_source AS ENUM ('js_exception','bugsnag','cloudwatch','datadog','newrelic','rollbar','sentry','stackdriver','sumologic'); - END IF; - END; - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'error_status') THEN - CREATE TYPE error_status AS ENUM ('unresolved','resolved','ignored'); - END IF; - END; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'error_source') THEN + CREATE TYPE error_source AS ENUM ('js_exception','bugsnag','cloudwatch','datadog','newrelic','rollbar','sentry','stackdriver','sumologic'); + END IF; + + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'error_status') THEN + CREATE TYPE error_status AS ENUM ('unresolved','resolved','ignored'); + END IF; + CREATE TABLE IF NOT EXISTS errors ( error_id text NOT NULL PRIMARY KEY, @@ -563,27 +552,26 @@ $$ ); CREATE INDEX IF NOT EXISTS user_viewed_errors_user_id_idx ON public.user_viewed_errors (user_id); CREATE INDEX IF NOT EXISTS user_viewed_errors_error_id_idx ON public.user_viewed_errors (error_id); - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'platform') THEN - CREATE TYPE platform AS ENUM ('web','ios','android'); - END IF; - END; - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'device_type') THEN - CREATE TYPE device_type AS ENUM ('desktop','tablet','mobile','other'); - END IF; - END; - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'country') THEN - CREATE TYPE country AS ENUM ('UN','RW','SO','YE','IQ','SA','IR','CY','TZ','SY','AM','KE','CD','DJ','UG','CF','SC','JO','LB','KW','OM','QA','BH','AE','IL','TR','ET','ER','EG','SD','GR','BI','EE','LV','AZ','LT','SJ','GE','MD','BY','FI','AX','UA','MK','HU','BG','AL','PL','RO','XK','ZW','ZM','KM','MW','LS','BW','MU','SZ','RE','ZA','YT','MZ','MG','AF','PK','BD','TM','TJ','LK','BT','IN','MV','IO','NP','MM','UZ','KZ','KG','TF','HM','CC','PW','VN','TH','ID','LA','TW','PH','MY','CN','HK','BN','MO','KH','KR','JP','KP','SG','CK','TL','RU','MN','AU','CX','MH','FM','PG','SB','TV','NR','VU','NC','NF','NZ','FJ','LY','CM','SN','CG','PT','LR','CI','GH','GQ','NG','BF','TG','GW','MR','BJ','GA','SL','ST','GI','GM','GN','TD','NE','ML','EH','TN','ES','MA','MT','DZ','FO','DK','IS','GB','CH','SE','NL','AT','BE','DE','LU','IE','MC','FR','AD','LI','JE','IM','GG','SK','CZ','NO','VA','SM','IT','SI','ME','HR','BA','AO','NA','SH','BV','BB','CV','GY','GF','SR','PM','GL','PY','UY','BR','FK','GS','JM','DO','CU','MQ','BS','BM','AI','TT','KN','DM','AG','LC','TC','AW','VG','VC','MS','MF','BL','GP','GD','KY','BZ','SV','GT','HN','NI','CR','VE','EC','CO','PA','HT','AR','CL','BO','PE','MX','PF','PN','KI','TK','TO','WF','WS','NU','MP','GU','PR','VI','UM','AS','CA','US','PS','RS','AQ','SX','CW','BQ','SS'); - END IF; - END; + + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'platform') THEN + CREATE TYPE platform AS ENUM ('web','ios','android'); + END IF; + + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'device_type') THEN + CREATE TYPE device_type AS ENUM ('desktop','tablet','mobile','other'); + END IF; + + + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'country') THEN + CREATE TYPE country AS ENUM ('UN','RW','SO','YE','IQ','SA','IR','CY','TZ','SY','AM','KE','CD','DJ','UG','CF','SC','JO','LB','KW','OM','QA','BH','AE','IL','TR','ET','ER','EG','SD','GR','BI','EE','LV','AZ','LT','SJ','GE','MD','BY','FI','AX','UA','MK','HU','BG','AL','PL','RO','XK','ZW','ZM','KM','MW','LS','BW','MU','SZ','RE','ZA','YT','MZ','MG','AF','PK','BD','TM','TJ','LK','BT','IN','MV','IO','NP','MM','UZ','KZ','KG','TF','HM','CC','PW','VN','TH','ID','LA','TW','PH','MY','CN','HK','BN','MO','KH','KR','JP','KP','SG','CK','TL','RU','MN','AU','CX','MH','FM','PG','SB','TV','NR','VU','NC','NF','NZ','FJ','LY','CM','SN','CG','PT','LR','CI','GH','GQ','NG','BF','TG','GW','MR','BJ','GA','SL','ST','GI','GM','GN','TD','NE','ML','EH','TN','ES','MA','MT','DZ','FO','DK','IS','GB','CH','SE','NL','AT','BE','DE','LU','IE','MC','FR','AD','LI','JE','IM','GG','SK','CZ','NO','VA','SM','IT','SI','ME','HR','BA','AO','NA','SH','BV','BB','CV','GY','GF','SR','PM','GL','PY','UY','BR','FK','GS','JM','DO','CU','MQ','BS','BM','AI','TT','KN','DM','AG','LC','TC','AW','VG','VC','MS','MF','BL','GP','GD','KY','BZ','SV','GT','HN','NI','CR','VE','EC','CO','PA','HT','AR','CL','BO','PE','MX','PF','PN','KI','TK','TO','WF','WS','NU','MP','GU','PR','VI','UM','AS','CA','US','PS','RS','AQ','SX','CW','BQ','SS'); + END IF; + CREATE TABLE IF NOT EXISTS sessions ( session_id bigint PRIMARY KEY, @@ -741,13 +729,12 @@ $$ CREATE TYPE job_status AS ENUM ('scheduled','running','cancelled','failed','completed'); END IF; END; - BEGIN - IF NOT EXISTS(SELECT * - FROM pg_type typ - WHERE typ.typname = 'job_action') THEN - CREATE TYPE job_action AS ENUM ('delete_user_data'); - END IF; - END; + IF NOT EXISTS(SELECT * + FROM pg_type typ + WHERE typ.typname = 'job_action') THEN + CREATE TYPE job_action AS ENUM ('delete_user_data'); + END IF; + CREATE TABLE IF NOT EXISTS jobs ( job_id integer generated BY DEFAULT AS IDENTITY PRIMARY KEY, From 6dbf80741d00a381d35918993a8f51b62763e8a7 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Wed, 26 Jan 2022 12:25:36 +0100 Subject: [PATCH 203/206] Update frontend.yaml --- .github/workflows/frontend.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 84af48e6a..8c5038f5f 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -1,4 +1,4 @@ -name: S3 Deploy +name: Frontend FOSS Deployment on: push: branches: @@ -27,8 +27,8 @@ jobs: method: kubeconfig kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. id: setcontext - - name: Install - run: npm install +# - name: Install +# run: npm install - name: Build and deploy run: | From 1899cc99bfc31121ce9f5abd7496708e055bbfbc Mon Sep 17 00:00:00 2001 From: Taha Yassine Kraiem Date: Wed, 26 Jan 2022 12:28:02 +0100 Subject: [PATCH 204/206] feat(frontend): small change to re-build --- frontend/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/build.sh b/frontend/build.sh index 010b69a34..98603cba1 100644 --- a/frontend/build.sh +++ b/frontend/build.sh @@ -28,4 +28,4 @@ function build(){ } check_prereq -build $1 +build $1 \ No newline at end of file From 52539e7aa23bb26ddbf9f578c4a431b9be97092d Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Wed, 26 Jan 2022 12:30:56 +0100 Subject: [PATCH 205/206] Delete frontend-ee.yaml --- .github/workflows/frontend-ee.yaml | 51 ------------------------------ 1 file changed, 51 deletions(-) delete mode 100644 .github/workflows/frontend-ee.yaml diff --git a/.github/workflows/frontend-ee.yaml b/.github/workflows/frontend-ee.yaml deleted file mode 100644 index f4de13db1..000000000 --- a/.github/workflows/frontend-ee.yaml +++ /dev/null @@ -1,51 +0,0 @@ -name: S3 Deploy EE -on: - push: - branches: - - dev - paths: - - ee/frontend/** - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Cache node modules - uses: actions/cache@v1 - with: - path: node_modules - key: ${{ runner.OS }}-build-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.OS }}-build- - ${{ runner.OS }}- - - - uses: azure/k8s-set-context@v1 - with: - method: kubeconfig - kubeconfig: ${{ secrets.EE_KUBECONFIG }} # Use content of kubeconfig in secret. - id: setcontext - - name: Install - run: npm install - - - name: Build and deploy - run: | - cd frontend - bash build.sh - cp -arl public frontend - minio_pod=$(kubectl get po -n db -l app.kubernetes.io/name=minio -n db --output custom-columns=name:.metadata.name | tail -n+2) - echo $minio_pod - echo copying frontend to container. - kubectl -n db cp frontend $minio_pod:/data/ - rm -rf frontend - - # - name: Debug Job - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # AWS_REGION: eu-central-1 - # AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} From 6277bf62185ad02dc0ff20e6bdd81fcccdf7a982 Mon Sep 17 00:00:00 2001 From: Mehdi Osman Date: Wed, 26 Jan 2022 12:34:26 +0100 Subject: [PATCH 206/206] Update build.sh --- frontend/build.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/frontend/build.sh b/frontend/build.sh index 98603cba1..7b656bc8f 100644 --- a/frontend/build.sh +++ b/frontend/build.sh @@ -18,14 +18,9 @@ check_prereq() { } function build(){ - # Copy enterprise code - [[ $1 == "ee" ]] && { - cp ../ee/frontend/* ./ - ee="true" - } # Run docker as the same user, else we'll run in to permission issues. docker run --rm -v /etc/passwd:/etc/passwd -u `id -u`:`id -g` -v $(pwd):/home/${USER} -w /home/${USER} --name node_build node:14-stretch-slim /bin/bash -c "npm install && npm run build:oss" } check_prereq -build $1 \ No newline at end of file +build $1