jan/openreplay
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/api-v1.10.0' into dev

Browse source
This commit is contained in:
Taha Yassine Kraiem 2023-02-20 11:42:53 +01:00
commit 08ad3afed2
5 changed files with 207 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.github/workflows/assist.yaml vendored
View file

@ -5,12 +5,11 @@ on:
branches:
- dev
paths:
- "ee/utilities/**"
- "utilities/*/**"
- "!utilities/.gitignore"
- "!utilities/*-dev.sh"
name: Build and Deploy Assist EE
name: Build and Deploy Assist
jobs:
deploy:
@ -21,7 +20,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
with:
# We need to diff with old commit
# We need to diff with old commit
# to see which workers got changed.
fetch-depth: 2
@ -39,12 +38,12 @@ jobs:
id: build-image
env:
DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }}
IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}-ee
IMAGE_TAG: ${{ github.ref_name }}_${{ github.sha }}
ENVIRONMENT: staging
run: |
skip_security_checks=${{ github.event.inputs.skip_security_checks }}
cd utilities
PUSH_IMAGE=0 bash -x ./build.sh ee
PUSH_IMAGE=0 bash -x ./build.sh
[[ "x$skip_security_checks" == "xtrue" ]] || {
curl -L https://github.com/aquasecurity/trivy/releases/download/v0.34.0/trivy_0.34.0_Linux-64bit.tar.gz | tar -xzf - -C ./
images=("assist")

3
ee/api/chalicelib/core/alerts_processor.py
View file

@ -149,8 +149,7 @@ def Build(a):
"startDate": TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000,
"timestamp_sub2": TimeUTC.now() - 2 * a["options"]["currentPeriod"] * 60 * 1000}
else:
sub1 = f"""{subQ} AND timestamp>=%(startDate)s
AND timestamp<=%(now)s
sub1 = f"""{subQ} {"AND timestamp >= %(startDate)s AND timestamp <= %(now)s" if not is_ss else ""}
{"AND start_ts >= %(startDate)s AND start_ts <= %(now)s" if j_s else ""}"""
params["startDate"] = TimeUTC.now() - a["options"]["currentPeriod"] * 60 * 1000
sub2 = f"""{subQ} {"AND timestamp < %(startDate)s AND timestamp >= %(timestamp_sub2)s" if not is_ss else ""}

3
ee/api/chalicelib/core/alerts_processor_exp.py
View file

@ -4,9 +4,10 @@ from decouple import config
import schemas
from chalicelib.core import alerts_listener, alerts_processor
from chalicelib.core import sessions, alerts
from chalicelib.core import alerts
from chalicelib.utils import pg_client, ch_client, exp_ch_helper
from chalicelib.utils.TimeUTC import TimeUTC
from chalicelib.core import sessions_exp as sessions
logging.basicConfig(level=config("LOGLEVEL", default=logging.INFO))

102
ee/scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql
View file

@ -138,9 +138,8 @@ ALTER TABLE IF EXISTS projects
ADD COLUMN IF NOT EXISTS beacon_size integer NOT NULL DEFAULT 0;
-- To migrate saved search data
-- SET client_min_messages TO NOTICE;
-- SET client_min_messages TO NOTICE;
SET client_min_messages TO NOTICE;
CREATE OR REPLACE FUNCTION get_new_event_key(key text)
RETURNS text AS
$$
@ -326,6 +325,105 @@ $$
$$
LANGUAGE plpgsql;
-- To migrate saved metric_series data
DO
$$
DECLARE
row RECORD;
events_att JSONB;
event_filters_att JSONB;
filters_att JSONB;
element JSONB;
s_element JSONB;
new_value TEXT;
new_events JSONB[];
new_filters JSONB[];
new_event_filters JSONB[];
changed BOOLEAN;
planned_update JSONB[];
BEGIN
planned_update := '{}'::jsonb[];
FOR row IN SELECT * FROM metric_series
LOOP
-- Transform events attributes
events_att := row.filter -> 'events';
IF events_att IS NOT NULL THEN
new_events := '{}'::jsonb[];
FOR element IN SELECT jsonb_array_elements(events_att)
LOOP
changed := FALSE;
new_value := get_new_event_key(element ->> 'type');
if new_value IS NOT NULL THEN
changed := TRUE;
new_value := replace(new_value, '"', '');
element := element || jsonb_build_object('type', new_value);
END IF;
-- Transform event's sub-filters attributes
event_filters_att := element -> 'filters';
new_event_filters := '{}'::jsonb[];
IF event_filters_att IS NOT NULL AND jsonb_array_length(event_filters_att) > 0 THEN
FOR s_element IN SELECT jsonb_array_elements(event_filters_att)
LOOP
new_value := get_new_event_filter_key(s_element ->> 'type');
if new_value IS NOT NULL THEN
changed := TRUE;
new_value := replace(new_value, '"', '');
s_element := s_element || jsonb_build_object('type', new_value);
new_event_filters := array_append(new_event_filters, s_element);
END IF;
END LOOP;
element := element || jsonb_build_object('filters', new_event_filters);
END IF;
IF changed THEN
new_events := array_append(new_events, element);
END IF;
END LOOP;
IF array_length(new_events, 1) > 0 THEN
row.filter := row.filter || jsonb_build_object('events', new_events);
END IF;
END IF;
-- Transform filters attributes
filters_att := row.filter -> 'filters';
IF filters_att IS NOT NULL THEN
new_filters := '{}'::jsonb;
FOR element IN SELECT jsonb_array_elements(filters_att)
LOOP
new_value := get_new_filter_key(element ->> 'type');
if new_value IS NOT NULL THEN
new_value := replace(new_value, '"', '');
element := element || jsonb_build_object('type', new_value);
new_filters := array_append(new_filters, element);
END IF;
END LOOP;
IF array_length(new_filters, 1) > 0 THEN
row.filter := row.filter || jsonb_build_object('filters', new_filters);
END IF;
END IF;
IF array_length(new_events, 1) > 0 OR array_length(new_filters, 1) > 0 THEN
planned_update := array_append(planned_update,
jsonb_build_object('id', row.series_id, 'change', row.filter));
END IF;
END LOOP;
-- Update metric_series
IF array_length(planned_update, 1) > 0 THEN
raise notice 'must update % elements',array_length(planned_update, 1);
UPDATE metric_series
SET filter=changes.change -> 'change'
FROM (SELECT unnest(planned_update)) AS changes(change)
WHERE series_id = (changes.change -> 'id')::integer;
raise notice 'update done';
ELSE
raise notice 'nothing to update';
END IF;
END ;
$$
LANGUAGE plpgsql;
DROP FUNCTION get_new_filter_key;
DROP FUNCTION get_new_event_filter_key;
DROP FUNCTION get_new_event_key;

102
scripts/schema/db/init_dbs/postgresql/1.10.0/1.10.0.sql
View file

@ -112,9 +112,8 @@ ALTER TABLE IF EXISTS projects
ADD COLUMN IF NOT EXISTS beacon_size integer NOT NULL DEFAULT 0;
-- To migrate saved search data
-- SET client_min_messages TO NOTICE;
-- SET client_min_messages TO NOTICE;
SET client_min_messages TO NOTICE;
CREATE OR REPLACE FUNCTION get_new_event_key(key text)
RETURNS text AS
$$
@ -300,6 +299,105 @@ $$
$$
LANGUAGE plpgsql;
-- To migrate saved metric_series data
DO
$$
DECLARE
row RECORD;
events_att JSONB;
event_filters_att JSONB;
filters_att JSONB;
element JSONB;
s_element JSONB;
new_value TEXT;
new_events JSONB[];
new_filters JSONB[];
new_event_filters JSONB[];
changed BOOLEAN;
planned_update JSONB[];
BEGIN
planned_update := '{}'::jsonb[];
FOR row IN SELECT * FROM metric_series
LOOP
-- Transform events attributes
events_att := row.filter -> 'events';
IF events_att IS NOT NULL THEN
new_events := '{}'::jsonb[];
FOR element IN SELECT jsonb_array_elements(events_att)
LOOP
changed := FALSE;
new_value := get_new_event_key(element ->> 'type');
if new_value IS NOT NULL THEN
changed := TRUE;
new_value := replace(new_value, '"', '');
element := element || jsonb_build_object('type', new_value);
END IF;
-- Transform event's sub-filters attributes
event_filters_att := element -> 'filters';
new_event_filters := '{}'::jsonb[];
IF event_filters_att IS NOT NULL AND jsonb_array_length(event_filters_att) > 0 THEN
FOR s_element IN SELECT jsonb_array_elements(event_filters_att)
LOOP
new_value := get_new_event_filter_key(s_element ->> 'type');
if new_value IS NOT NULL THEN
changed := TRUE;
new_value := replace(new_value, '"', '');
s_element := s_element || jsonb_build_object('type', new_value);
new_event_filters := array_append(new_event_filters, s_element);
END IF;
END LOOP;
element := element || jsonb_build_object('filters', new_event_filters);
END IF;
IF changed THEN
new_events := array_append(new_events, element);
END IF;
END LOOP;
IF array_length(new_events, 1) > 0 THEN
row.filter := row.filter || jsonb_build_object('events', new_events);
END IF;
END IF;
-- Transform filters attributes
filters_att := row.filter -> 'filters';
IF filters_att IS NOT NULL THEN
new_filters := '{}'::jsonb;
FOR element IN SELECT jsonb_array_elements(filters_att)
LOOP
new_value := get_new_filter_key(element ->> 'type');
if new_value IS NOT NULL THEN
new_value := replace(new_value, '"', '');
element := element || jsonb_build_object('type', new_value);
new_filters := array_append(new_filters, element);
END IF;
END LOOP;
IF array_length(new_filters, 1) > 0 THEN
row.filter := row.filter || jsonb_build_object('filters', new_filters);
END IF;
END IF;
IF array_length(new_events, 1) > 0 OR array_length(new_filters, 1) > 0 THEN
planned_update := array_append(planned_update,
jsonb_build_object('id', row.series_id, 'change', row.filter));
END IF;
END LOOP;
-- Update metric_series
IF array_length(planned_update, 1) > 0 THEN
raise notice 'must update % elements',array_length(planned_update, 1);
UPDATE metric_series
SET filter=changes.change -> 'change'
FROM (SELECT unnest(planned_update)) AS changes(change)
WHERE series_id = (changes.change -> 'id')::integer;
raise notice 'update done';
ELSE
raise notice 'nothing to update';
END IF;
END ;
$$
LANGUAGE plpgsql;
DROP FUNCTION get_new_filter_key;
DROP FUNCTION get_new_event_filter_key;
DROP FUNCTION get_new_event_key;