Self hosted release 0.21.0

orb-community · Nov 28, 2022 · 29455c2 · 29455c2
2 parents 7a84657 + 6728a47
commit 29455c2
Show file tree

Hide file tree

Showing 212 changed files with 8,253 additions and 2,513 deletions.
diff --git a/.github/workflows/deploy-stg.yml b/.github/workflows/deploy-stg.yml
diff --git a/.github/workflows/go-develop.yml b/.github/workflows/go-develop.yml
@@ -41,6 +41,8 @@ jobs:
               - 'cmd/sinks/**'
               - 'sinker/**'
               - 'cmd/sinker/**'
+              - 'maestro/**'
+              - 'cmd/maestro/**'
             ui: 
               - 'ui/**'
             docs:
@@ -81,10 +83,14 @@ jobs:
         uses: actions/checkout@v2
         with:
           name: workspace
+
       - name: Run go report
         uses: ./.github/actions/go-report
         with:
           go_report_threshold: 90.1 #grade A+
+
+      - name: refresh go-report
+        uses: creekorful/[email protected]
 
   update-api-docs:
     needs: prebuild
@@ -111,7 +117,31 @@ jobs:
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.agent == 'true' }}
-        run: SERVICE=agent make test_service
+        run: |
+          SERVICE=agent make test_service_cov
+        
+      - name: Install dependencies
+        if: ${{ needs.prebuild.outputs.agent == 'true' }}        
+        run: |
+          go mod tidy
+          sudo apt update && sudo apt install -y build-essential jq
+          go install github.com/axw/gocov/gocov@latest
+          go install github.com/AlekSi/gocov-xml@latest
+          
+      - name: coverage to xml
+        if: ${{ needs.prebuild.outputs.agent == 'true' }}
+        run: |
+          echo "Current directory: ${PWD}"
+          echo "GITHUB_PR_ISSUE_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" >> $GITHUB_ENV
+          gocov convert ./coverage.out | gocov-xml > ./coverage.xml
+          
+      - name: Upload coverage to Codecov
+        if: ${{ needs.prebuild.outputs.agent == 'true' }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          name: orb
+          verbose: true
 
   test-fleet:
     runs-on: ubuntu-latest
@@ -124,10 +154,32 @@ jobs:
           go-version: 1.19
 
       - name: Go unit tests
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
         run: |
-          if [ "${{ needs.prebuild.outputs.orb }}" == "true" ]; then
-            SERVICE=fleet make test_service
-          fi
+            SERVICE=fleet make test_service_cov
+          
+      - name: Install dependencies
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          go mod tidy
+          sudo apt update && sudo apt install -y build-essential jq
+          go install github.com/axw/gocov/gocov@latest
+          go install github.com/AlekSi/gocov-xml@latest
+          
+      - name: coverage to xml
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          echo "Current directory: ${PWD}"
+          echo "GITHUB_PR_ISSUE_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" >> $GITHUB_ENV
+          gocov convert ./coverage.out | gocov-xml > ./coverage.xml
+          
+      - name: Upload coverage to Codecov
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          name: orb
+          verbose: true
 
   test-policies:
     runs-on: ubuntu-latest
@@ -141,7 +193,31 @@ jobs:
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
-        run: SERVICE=policies make test_service
+        run: |
+          SERVICE=policies make test_service_cov
+          
+      - name: Install dependencies
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          go mod tidy
+          sudo apt update && sudo apt install -y build-essential jq
+          go install github.com/axw/gocov/gocov@latest
+          go install github.com/AlekSi/gocov-xml@latest
+          
+      - name: coverage to xml
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          echo "Current directory: ${PWD}"
+          echo "GITHUB_PR_ISSUE_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" >> $GITHUB_ENV
+          gocov convert ./coverage.out | gocov-xml > ./coverage.xml
+          
+      - name: Upload coverage to Codecov
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          name: orb
+          verbose: true
 
   test-sinks:
     runs-on: ubuntu-latest
@@ -155,7 +231,31 @@ jobs:
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
-        run: SERVICE=sinks make test_service
+        run: |
+          SERVICE=sinks make test_service_cov
+          
+      - name: Install dependencies
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          go mod tidy
+          sudo apt update && sudo apt install -y build-essential jq
+          go install github.com/axw/gocov/gocov@latest
+          go install github.com/AlekSi/gocov-xml@latest
+          
+      - name: coverage to xml
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          echo "Current directory: ${PWD}"
+          echo "GITHUB_PR_ISSUE_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" >> $GITHUB_ENV
+          gocov convert ./coverage.out | gocov-xml > ./coverage.xml
+          
+      - name: Upload coverage to Codecov
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          name: orb
+          verbose: true
 
   test-sinker:
     runs-on: ubuntu-latest
@@ -169,7 +269,69 @@ jobs:
 
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
-        run: SERVICE=sinker make test_service
+        run: |
+          SERVICE=sinker make test_service_cov
+          
+      - name: Install dependencies
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          go mod tidy
+          sudo apt update && sudo apt install -y build-essential jq
+          go install github.com/axw/gocov/gocov@latest
+          go install github.com/AlekSi/gocov-xml@latest
+          
+      - name: coverage to xml
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          echo "Current directory: ${PWD}"
+          echo "GITHUB_PR_ISSUE_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" >> $GITHUB_ENV
+          gocov convert ./coverage.out | gocov-xml > ./coverage.xml
+          
+      - name: Upload coverage to Codecov
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          name: orb
+          verbose: true   
+
+  test-maestro:
+    runs-on: ubuntu-latest
+    needs: prebuild
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
+      - name: Go unit tests
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+            SERVICE=maestro make test_service_cov
+          
+      - name: Install dependencies
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          go mod tidy
+          sudo apt update && sudo apt install -y build-essential jq
+          go install github.com/axw/gocov/gocov@latest
+          go install github.com/AlekSi/gocov-xml@latest
+          
+      - name: coverage to xml
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        run: |
+          echo "Current directory: ${PWD}"
+          echo "GITHUB_PR_ISSUE_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")" >> $GITHUB_ENV
+          gocov convert ./coverage.out | gocov-xml > ./coverage.xml
+          
+      - name: Upload coverage to Codecov
+        if: ${{ needs.prebuild.outputs.orb == 'true' }}
+        uses: codecov/codecov-action@v3
+        with:
+          files: coverage.xml
+          name: orb
+          verbose: true
 
   package-agent:
     needs:
@@ -333,6 +495,33 @@ jobs:
       - name: Push service containers
         if: github.event_name != 'pull_request'
         run: docker push -a ns1labs/orb-sinks
+
+  package-maestro:
+    needs:
+      - prebuild
+      - test-maestro
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get short commit hash to a variable
+        id: commit_hash
+        run: |
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+          echo ${{ needs.prebuild.outputs.VERSION }} > VERSION
+
+      - name: Build service containers
+        run: SERVICE=maestro make build_docker
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push service containers
+        if: github.event_name != 'pull_request'
+        run: docker push -a ns1labs/orb-maestro
 
   package-ui-dependencies:
     needs:

diff --git a/.github/workflows/go-production.yml b/.github/workflows/go-production.yml
@@ -37,6 +37,8 @@ jobs:
               - 'cmd/sinks/**'
               - 'sinker/**'
               - 'cmd/sinker/**'
+              - 'maestro/**'
+              - 'cmd/maestro/**'
             ui: 
               - 'ui/**'
       - name: Set branch name
@@ -131,6 +133,22 @@ jobs:
       - name: Go unit tests
         if: ${{ needs.prebuild.outputs.orb == 'true' }}
         run: SERVICE=sinker make test_service
+
+  test-maestro:
+    runs-on: ubuntu-latest
+    needs: prebuild
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
+      - name: Go unit tests
+        run: |
+          if [ "${{ needs.prebuild.outputs.orb }}" == "true" ]; then
+            SERVICE=maestro make test_service
+          fi
 
   package-agent:
 # This is just for debug agent
@@ -288,6 +306,32 @@ jobs:
       - name: Push service containers
         if: github.event_name != 'pull_request'
         run: docker push -a ns1labs/orb-sinks
+
+  package-maestro:
+    needs:
+      - prebuild
+      - test-maestro
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get short commit hash to a variable
+        id: commit_hash
+        run: |
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+          echo ${{ needs.prebuild.outputs.VERSION }} > VERSION
+      - name: Build service containers
+        run: SERVICE=maestro make build_docker
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push service containers
+        if: github.event_name != 'pull_request'
+        run: docker push -a ns1labs/orb-maestro
 
   package-ui-dependencies:
     needs:

diff --git a/Makefile b/Makefile
@@ -40,6 +40,10 @@ define run_test
 	 go test -mod=mod -race -count 1 -tags test $(shell go list ./... | grep -v 'cmd' | grep '$(SERVICE)')
 endef
 
+define run_test_coverage
+	 go test -mod=mod -race -count 1 -tags test -cover -coverprofile=coverage.out -covermode=atomic $(shell go list ./... | grep -v 'cmd' | grep '$(SERVICE)')
+endef
+
 define make_docker
 	$(eval SERVICE=$(shell [ -z "$(SERVICE)" ] && echo $(subst docker_,,$(1)) || echo $(SERVICE)))
 	docker build \
@@ -87,9 +91,14 @@ test:
 
 run_test_service: test_service $(2)
 
+run_test_service_cov: test_service_cov $(2)
+
 test_service:
 	$(call run_test,$(@))
 
+test_service_cov:
+	$(call run_test_coverage,$(@))
+
 proto:
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative policies/pb/policies.proto
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative fleet/pb/fleet.proto