From 6e60fe2c61dc0aba4c1d9ec309f634fe30d99aac Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 22:54:41 +0530 Subject: [PATCH 1/9] chore(installation): cleanup unused values Signed-off-by: Rajesh Rajendran --- scripts/helm/vars.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/helm/vars.yaml b/scripts/helm/vars.yaml index 89c182378..bc6cc51e2 100644 --- a/scripts/helm/vars.yaml +++ b/scripts/helm/vars.yaml @@ -7,7 +7,7 @@ # Give absolute file path. # Use following command to get the full file path # `readlink -f ` -kubeconfig_path: /home/rajeshr/.cred/asayer/kube-oss.yaml +kubeconfig_path: "" ################### ## Optional Fields. @@ -36,8 +36,8 @@ domain_name: "" # By Default, we'll create a self signed certificate for nginx, and populate the values here. # Once you've proper domain name, and ssl certificate # Change the following variables accordingly. -nginx_ssl_cert_file_path: "/home/rajesh/Documents/projects/asayer/asayer-os/scripts/helm/nginx-ingress/site.crt" -nginx_ssl_key_file_path: "/home/rajesh/Documents/projects/asayer/asayer-os/scripts/helm/nginx-ingress/site.key" +nginx_ssl_cert_file_path: "" +nginx_ssl_key_file_path: "" # Enable monitoring # If set, monitoring stack will be installed @@ -50,5 +50,5 @@ enable_monitoring: "false" # If not defined, will generate at runtime. # Use following command to generate passwordwill give # `openssl rand -base64 30` -minio_access_key: "YkkPAPYjogRlicqvCuNSHkfsdGtCCq" -minio_secret_key: "MSVmVGXfTpVNKUfVYdrKQemekoFeUg" +minio_access_key: "" +minio_secret_key: "" From 46db7efa8500071acf33c39fd9bd0c9713969baa Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Mon, 10 May 2021 19:42:42 +0200 Subject: [PATCH 2/9] fix (backend): build script --- backend/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/build.sh b/backend/build.sh index 3b3014282..da197a9e8 100644 --- a/backend/build.sh +++ b/backend/build.sh @@ -21,7 +21,7 @@ check_prereq() { function build_api(){ # Copy enterprize code [[ $1 == "ee" ]] && { - cp ../ee/backend/* ./ + cp -r ../ee/backend/* ./ ee="true" } [[ $2 != "" ]] && { From f2e5387af2018d39fc0ee6ffcfd26f03b18f094f Mon Sep 17 00:00:00 2001 From: ShiKhu Date: Mon, 10 May 2021 19:44:28 +0200 Subject: [PATCH 3/9] feat (backend ee): check license --- ee/backend/pkg/db/clickhouse/connector.go | 4 ++ ee/backend/pkg/license/check.go | 61 +++++++++++++++++++++++ ee/backend/pkg/queue/import.go | 3 ++ ee/backend/services/db/stats.go | 4 +- 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 ee/backend/pkg/license/check.go diff --git a/ee/backend/pkg/db/clickhouse/connector.go b/ee/backend/pkg/db/clickhouse/connector.go index 0851ac76a..0ff49a06a 100644 --- a/ee/backend/pkg/db/clickhouse/connector.go +++ b/ee/backend/pkg/db/clickhouse/connector.go @@ -4,6 +4,8 @@ import ( "log" "database/sql" _ "github.com/ClickHouse/clickhouse-go" + + "openreplay/backend/pkg/license" ) type Connector struct { @@ -27,6 +29,8 @@ type Connector struct { } func NewConnector(url string) *Connector { + license.CheckLicense() + db, err := sql.Open("clickhouse", url) if err != nil { log.Fatalln(err) diff --git a/ee/backend/pkg/license/check.go b/ee/backend/pkg/license/check.go new file mode 100644 index 000000000..6b33a625e --- /dev/null +++ b/ee/backend/pkg/license/check.go @@ -0,0 +1,61 @@ +package license + +import ( + "log" + "net/http" + "encoding/json" + "io/ioutil" + "bytes" + + "openreplay/backend/pkg/env" +) + + + +type request struct { + MID string `json:"mid"` + License string `json:"license"` +} + +type response struct { + Data struct { + IsValid bool `json:"valid"` + ExpirationTimestamp int64 `json:"expiration"` + } `json:"data"` +} + + +func CheckLicense() { + license := env.String("LICENSE_KEY") + + requestBody, err := json.Marshal(request{ License: license }) + if err != nil { + log.Fatal("Can not form a license check request.") + } + + resp, err := http.Post("https://parrot.asayer.io/os/license", "application/json", bytes.NewReader(requestBody)) + if err != nil { + log.Fatalf("Error while checking license. %v", err) + } + + if resp.StatusCode != 200 { + log.Fatal("Error on license check request.") + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatalf("Error while reading license check response. %v", err) + } + + respJson := new(response) + if err = json.Unmarshal(body, respJson); err != nil { + log.Fatalf("Error while parsing license check response. %v", err) + } + + if !respJson.Data.IsValid { + log.Fatal("License is not valid.") + } + + +} \ No newline at end of file diff --git a/ee/backend/pkg/queue/import.go b/ee/backend/pkg/queue/import.go index d1443c40e..abff07e9a 100644 --- a/ee/backend/pkg/queue/import.go +++ b/ee/backend/pkg/queue/import.go @@ -3,13 +3,16 @@ package queue import ( "openreplay/backend/pkg/kafka" "openreplay/backend/pkg/queue/types" + "openreplay/backend/pkg/license" ) func NewConsumer(group string, topics []string, handler types.MessageHandler) types.Consumer { + license.CheckLicense() return kafka.NewConsumer(group, topics, handler) } func NewProducer() types.Producer { + license.CheckLicense() return kafka.NewProducer() } diff --git a/ee/backend/services/db/stats.go b/ee/backend/services/db/stats.go index b2fcb3f67..67825e2a2 100644 --- a/ee/backend/services/db/stats.go +++ b/ee/backend/services/db/stats.go @@ -12,7 +12,7 @@ import ( ) var ch *clickhouse.Connector -var finalizeTicker *time.Ticker +var finalizeTicker <-chan time.Time func initStats() { ch = clickhouse.NewConnector(env.String("CLICKHOUSE_STRING")) @@ -20,7 +20,7 @@ func initStats() { log.Fatalf("Clickhouse prepare error: %v\n", err) } - finalizeTicker = time.NewTicker(20 * time.Minute) + finalizeTicker = time.Tick(20 * time.Minute) } From aaa947862ca48c1caa0a5b83dbdb6d650d3d9a4f Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 23:19:46 +0530 Subject: [PATCH 4/9] ci(deployment): fix kube deployment Signed-off-by: Rajesh Rajendran --- .github/workflows/api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 03012e1c4..435d07126 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -45,7 +45,7 @@ jobs: } - name: Deploy to kubernetes run: | - cd ../scripts/helm/ + cd scripts/helm/ sed -i "s#kubeconfig.*#kubeconfig_path: ${KUBECONFIG}#g" vars.yaml sed -i "s/tag:.*/tag: \"$IMAGE_TAG\"/g" app/chalice.yaml bash kube-install.sh --app chalice From fc8328629fabae7bb8be4bb5859e28b96a1cbfaa Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 23:22:24 +0530 Subject: [PATCH 5/9] Revert "tmp: enabling debug." This reverts commit f94610f6235021a3f68d40c9d12ccb93e7d81e90. --- .github/workflows/workers.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/workers.yaml b/.github/workflows/workers.yaml index 98e31c2cc..c3a30c09b 100644 --- a/.github/workflows/workers.yaml +++ b/.github/workflows/workers.yaml @@ -72,11 +72,11 @@ jobs: bash kube-install.sh --app $image done - - name: Debug Job - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - env: - DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} - IMAGE_TAG: ${{ github.sha }} - ENVIRONMENT: staging - + # - name: Debug Job + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # env: + # DOCKER_REPO: ${{ secrets.OSS_REGISTRY_URL }} + # IMAGE_TAG: ${{ github.sha }} + # ENVIRONMENT: staging + # From fa7717f869f223b313c06daca30268d6fdb6d6be Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 23:23:12 +0530 Subject: [PATCH 6/9] Revert "ci(tmp): empty commit to trigger build." This reverts commit d187636d8e07d1b7a3513ef5867d8504b7aa0e1a. --- backend/services/assets/main.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/services/assets/main.go b/backend/services/assets/main.go index 9b0703ef2..0193be7bb 100644 --- a/backend/services/assets/main.go +++ b/backend/services/assets/main.go @@ -15,11 +15,11 @@ import ( "openreplay/backend/services/assets/cacher" ) -// empty commit to trigger build. + func main() { log.SetFlags(log.LstdFlags | log.LUTC | log.Llongfile) - GROUP_CACHE := env.String("GROUP_CACHE") + GROUP_CACHE := env.String("GROUP_CACHE") TOPIC_TRIGGER := env.String("TOPIC_TRIGGER") cacher := cacher.NewCacher( @@ -30,10 +30,10 @@ func main() { ) consumer := queue.NewMessageConsumer( - GROUP_CACHE, - []string{TOPIC_TRIGGER}, + GROUP_CACHE, + []string{ TOPIC_TRIGGER }, func(sessionID uint64, message messages.Message, e *types.Meta) { - switch msg := message.(type) { + switch msg := message.(type) { case *messages.AssetCache: cacher.CacheURL(sessionID, msg.URL) case *messages.ErrorEvent: @@ -48,14 +48,15 @@ func main() { for _, source := range sourceList { cacher.CacheJSFile(source) } - } + } }, ) + tick := time.Tick(20 * time.Minute) sigchan := make(chan os.Signal, 1) - signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) for { select { @@ -71,4 +72,4 @@ func main() { } } } -} +} \ No newline at end of file From 46d5bbd783ca0180b4f6231e7ddd6b4a42a8a540 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 23:30:02 +0530 Subject: [PATCH 7/9] ci(frontend): fixing deploying to kubernetes Signed-off-by: Rajesh Rajendran --- .github/workflows/frontend.yaml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 7e42967ec..77e4abfa0 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -22,26 +22,22 @@ jobs: ${{ runner.OS }}-build- ${{ runner.OS }}- + - uses: azure/k8s-set-context@v1 + with: + method: kubeconfig + kubeconfig: ${{ secrets.OSS_KUBECONFIG }} # Use content of kubeconfig in secret. + id: setcontext - name: Install run: npm install - - name: Build - run: npm run build:staging - env: - ENVIRONMENT: staging - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: eu-central-1 - AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }} + - name: Build and deploy run: | - aws configure set default.s3.signature_version s3v4 - aws --endpoint-url https://${{secrets.DOMAIN_NAME}}/frontend/ s3 cp \ - --recursive \ - --region "$AWS_REGION" \ - public s3://$AWS_S3_BUCKET_NAME + cd frontend + bash build.sh + cp -arl public frontend + minio_pod=$(kubectl get po -n db -l app.kubernetes.io/name=minio -n db --output custom-columns=name:.metadata.name | tail -n+2) + kubectl -n db cp frontend $minio_pod:/data/ + rm -rf frontend # - name: Debug Job # if: ${{ failure() }} From 304e0642f5d67b0ac16dda797a9d8de1ba7ce703 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 23:33:14 +0530 Subject: [PATCH 8/9] ci(workers): display which image to build Signed-off-by: Rajesh Rajendran --- .github/workflows/workers.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workers.yaml b/.github/workflows/workers.yaml index c3a30c09b..ef5e4f5b9 100644 --- a/.github/workflows/workers.yaml +++ b/.github/workflows/workers.yaml @@ -55,7 +55,8 @@ jobs: cd backend for image in $(cat images_to_build.txt); do - bash ./build.sh skip $image + echo "Bulding $image" + bash -x ./build.sh skip $image docker push $DOCKER_REPO/$image:$IMAGE_TAG echo "::set-output name=image::$DOCKER_REPO/$image:$IMAGE_TAG" done From e3877bd9d8c4936c1bcfb9bf2087bd0e76967400 Mon Sep 17 00:00:00 2001 From: Rajesh Rajendran Date: Mon, 10 May 2021 23:46:09 +0530 Subject: [PATCH 9/9] ci(front): verbose logs Signed-off-by: Rajesh Rajendran --- .github/workflows/frontend.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/frontend.yaml b/.github/workflows/frontend.yaml index 77e4abfa0..84af48e6a 100644 --- a/.github/workflows/frontend.yaml +++ b/.github/workflows/frontend.yaml @@ -36,6 +36,8 @@ jobs: bash build.sh cp -arl public frontend minio_pod=$(kubectl get po -n db -l app.kubernetes.io/name=minio -n db --output custom-columns=name:.metadata.name | tail -n+2) + echo $minio_pod + echo copying frontend to container. kubectl -n db cp frontend $minio_pod:/data/ rm -rf frontend