diff --git a/.github/workflows/assist.yaml b/.github/workflows/assist.yaml index cf4d184cf..c599d5cbd 100644 --- a/.github/workflows/assist.yaml +++ b/.github/workflows/assist.yaml @@ -5,12 +5,11 @@ on: branches: - dev paths: - - "ee/utilities/**" - "utilities/*/**" - "!utilities/.gitignore" - "!utilities/*-dev.sh" -name: Build and Deploy Assist EE +name: Build and Deploy Assist jobs: deploy: @@ -21,7 +20,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 with: - # We need to diff with old commit + # We need to diff with old commit # to see which workers got changed. fetch-depth: 2 @@ -39,12 +38,12 @@ jobs: id: build-image env: DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee + IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }} ENVIRONMENT: staging run: | skip_security_checks=${{ github.event.inputs.skip_security_checks }} cd utilities - PUSH_IMAGE=0 bash -x ./build.sh ee + PUSH_IMAGE=0 bash -x ./build.sh [[ "x$skip_security_checks" == "xtrue" ]] || { curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./ images=("assist") diff --git a/ee/api/chalicelib/core/alerts_processor.py b/ee/api/chalicelib/core/alerts_processor.py index 06663336c..17e4d275f 100644 --- a/ee/api/chalicelib/core/alerts_processor.py +++ b/ee/api/chalicelib/core/alerts_processor.py @@ -149,8 +149,7 @@ def Build(a): "startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000, "timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000} else: - sub1 = f"""{subQ} AND timestamp>=%(startDate)s - AND timestamp<=%(now)s + sub1 = f"""{subQ} {"AND timestamp >= %(startDate)s AND timestamp <= %(now)s" if not is_ss else ""} {"AND start_ts >= %(startDate)s AND start_ts <= %(now)s" if j_s else ""}""" params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000 sub2 = f"""{subQ} {"AND timestamp < %(startDate)s AND timestamp >= %(timestamp_sub2)s" if not is_ss else ""} diff --git a/ee/api/chalicelib/core/alerts_processor_exp.py b/ee/api/chalicelib/core/alerts_processor_exp.py index 37a1b843f..310c6faa9 100644 --- a/ee/api/chalicelib/core/alerts_processor_exp.py +++ b/ee/api/chalicelib/core/alerts_processor_exp.py @@ -4,9 +4,10 @@ from decouple import config import schemas from chalicelib.core import alerts_listener, alerts_processor -from chalicelib.core import sessions, alerts +from chalicelib.core import alerts from chalicelib.utils import pg_client, ch_client, exp_ch_helper from chalicelib.utils.TimeUTC import TimeUTC +from chalicelib.core import sessions_exp as sessions logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO)) diff --git a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index 30961fc88..68b115e46 100644 --- a/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -138,9 +138,8 @@ ALTER TABLE IF EXISTS projects ADD COLUMN IF NOT EXISTS beacon_size integer NOT NULL DEFAULT 0; -- To migrate saved search data --- SET client_min_messages TO NOTICE; --- SET client_min_messages TO NOTICE; +SET client_min_messages TO NOTICE; CREATE OR REPLACE FUNCTION get_new_event_key(key text) RETURNS text AS $$ @@ -326,6 +325,105 @@ $$ $$ LANGUAGE plpgsql; + +-- To migrate saved metric_series data +DO +$$ + DECLARE + row RECORD; + events_att JSONB; + event_filters_att JSONB; + filters_att JSONB; + element JSONB; + s_element JSONB; + new_value TEXT; + new_events JSONB[]; + new_filters JSONB[]; + new_event_filters JSONB[]; + changed BOOLEAN; + planned_update JSONB[]; + BEGIN + planned_update := '{}'::jsonb[]; + FOR row IN SELECT * FROM metric_series + LOOP + -- Transform events attributes + events_att := row.filter -> 'events'; + IF events_att IS NOT NULL THEN + new_events := '{}'::jsonb[]; + FOR element IN SELECT jsonb_array_elements(events_att) + LOOP + changed := FALSE; + new_value := get_new_event_key(element ->> 'type'); + if new_value IS NOT NULL THEN + changed := TRUE; + new_value := replace(new_value, '"', ''); + element := element || jsonb_build_object('type', new_value); + END IF; + -- Transform event's sub-filters attributes + event_filters_att := element -> 'filters'; + new_event_filters := '{}'::jsonb[]; + IF event_filters_att IS NOT NULL AND jsonb_array_length(event_filters_att) > 0 THEN + FOR s_element IN SELECT jsonb_array_elements(event_filters_att) + LOOP + new_value := get_new_event_filter_key(s_element ->> 'type'); + if new_value IS NOT NULL THEN + changed := TRUE; + new_value := replace(new_value, '"', ''); + s_element := s_element || jsonb_build_object('type', new_value); + new_event_filters := array_append(new_event_filters, s_element); + END IF; + END LOOP; + element := element || jsonb_build_object('filters', new_event_filters); + END IF; + IF changed THEN + new_events := array_append(new_events, element); + END IF; + END LOOP; + IF array_length(new_events, 1) > 0 THEN + row.filter := row.filter || jsonb_build_object('events', new_events); + END IF; + END IF; + + -- Transform filters attributes + filters_att := row.filter -> 'filters'; + IF filters_att IS NOT NULL THEN + new_filters := '{}'::jsonb; + FOR element IN SELECT jsonb_array_elements(filters_att) + LOOP + new_value := get_new_filter_key(element ->> 'type'); + if new_value IS NOT NULL THEN + new_value := replace(new_value, '"', ''); + element := element || jsonb_build_object('type', new_value); + new_filters := array_append(new_filters, element); + END IF; + END LOOP; + IF array_length(new_filters, 1) > 0 THEN + row.filter := row.filter || jsonb_build_object('filters', new_filters); + END IF; + END IF; + + IF array_length(new_events, 1) > 0 OR array_length(new_filters, 1) > 0 THEN + planned_update := array_append(planned_update, + jsonb_build_object('id', row.series_id, 'change', row.filter)); + END IF; + END LOOP; + + -- Update metric_series + IF array_length(planned_update, 1) > 0 THEN + raise notice 'must update % elements',array_length(planned_update, 1); + + UPDATE metric_series + SET filter=changes.change -> 'change' + FROM (SELECT unnest(planned_update)) AS changes(change) + WHERE series_id = (changes.change -> 'id')::integer; + raise notice 'update done'; + ELSE + raise notice 'nothing to update'; + END IF; + END ; +$$ +LANGUAGE plpgsql; + DROP FUNCTION get_new_filter_key; DROP FUNCTION get_new_event_filter_key; DROP FUNCTION get_new_event_key; diff --git a/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql index 76d6dd88b..b9f0380c4 100644 --- a/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql +++ b/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql @@ -112,9 +112,8 @@ ALTER TABLE IF EXISTS projects ADD COLUMN IF NOT EXISTS beacon_size integer NOT NULL DEFAULT 0; -- To migrate saved search data --- SET client_min_messages TO NOTICE; --- SET client_min_messages TO NOTICE; +SET client_min_messages TO NOTICE; CREATE OR REPLACE FUNCTION get_new_event_key(key text) RETURNS text AS $$ @@ -300,6 +299,105 @@ $$ $$ LANGUAGE plpgsql; + +-- To migrate saved metric_series data +DO +$$ + DECLARE + row RECORD; + events_att JSONB; + event_filters_att JSONB; + filters_att JSONB; + element JSONB; + s_element JSONB; + new_value TEXT; + new_events JSONB[]; + new_filters JSONB[]; + new_event_filters JSONB[]; + changed BOOLEAN; + planned_update JSONB[]; + BEGIN + planned_update := '{}'::jsonb[]; + FOR row IN SELECT * FROM metric_series + LOOP + -- Transform events attributes + events_att := row.filter -> 'events'; + IF events_att IS NOT NULL THEN + new_events := '{}'::jsonb[]; + FOR element IN SELECT jsonb_array_elements(events_att) + LOOP + changed := FALSE; + new_value := get_new_event_key(element ->> 'type'); + if new_value IS NOT NULL THEN + changed := TRUE; + new_value := replace(new_value, '"', ''); + element := element || jsonb_build_object('type', new_value); + END IF; + -- Transform event's sub-filters attributes + event_filters_att := element -> 'filters'; + new_event_filters := '{}'::jsonb[]; + IF event_filters_att IS NOT NULL AND jsonb_array_length(event_filters_att) > 0 THEN + FOR s_element IN SELECT jsonb_array_elements(event_filters_att) + LOOP + new_value := get_new_event_filter_key(s_element ->> 'type'); + if new_value IS NOT NULL THEN + changed := TRUE; + new_value := replace(new_value, '"', ''); + s_element := s_element || jsonb_build_object('type', new_value); + new_event_filters := array_append(new_event_filters, s_element); + END IF; + END LOOP; + element := element || jsonb_build_object('filters', new_event_filters); + END IF; + IF changed THEN + new_events := array_append(new_events, element); + END IF; + END LOOP; + IF array_length(new_events, 1) > 0 THEN + row.filter := row.filter || jsonb_build_object('events', new_events); + END IF; + END IF; + + -- Transform filters attributes + filters_att := row.filter -> 'filters'; + IF filters_att IS NOT NULL THEN + new_filters := '{}'::jsonb; + FOR element IN SELECT jsonb_array_elements(filters_att) + LOOP + new_value := get_new_filter_key(element ->> 'type'); + if new_value IS NOT NULL THEN + new_value := replace(new_value, '"', ''); + element := element || jsonb_build_object('type', new_value); + new_filters := array_append(new_filters, element); + END IF; + END LOOP; + IF array_length(new_filters, 1) > 0 THEN + row.filter := row.filter || jsonb_build_object('filters', new_filters); + END IF; + END IF; + + IF array_length(new_events, 1) > 0 OR array_length(new_filters, 1) > 0 THEN + planned_update := array_append(planned_update, + jsonb_build_object('id', row.series_id, 'change', row.filter)); + END IF; + END LOOP; + + -- Update metric_series + IF array_length(planned_update, 1) > 0 THEN + raise notice 'must update % elements',array_length(planned_update, 1); + + UPDATE metric_series + SET filter=changes.change -> 'change' + FROM (SELECT unnest(planned_update)) AS changes(change) + WHERE series_id = (changes.change -> 'id')::integer; + raise notice 'update done'; + ELSE + raise notice 'nothing to update'; + END IF; + END ; +$$ +LANGUAGE plpgsql; + DROP FUNCTION get_new_filter_key; DROP FUNCTION get_new_event_filter_key; DROP FUNCTION get_new_event_key;