Introduce batched services. #1836
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build installers | |
| on: [push, pull_request] | |
| jobs: | |
| build-test-linux: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| MYSQL_TEST_USER: root | |
| MYSQL_TEST_PASS: root | |
| MYSQL_TEST_ADDR: 127.0.0.1:3306 | |
| MYSQL_TEST_E2E_DB: e2e_test_db | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up MySQL | |
| run: | | |
| sudo /etc/init.d/mysql start | |
| mysql -e 'CREATE DATABASE ${{ env.MYSQL_TEST_E2E_DB }};' -u${{ env.MYSQL_TEST_USER }} -p${{ env.MYSQL_TEST_PASS }} | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.22' | |
| - name: Set up Go | |
| run: | | |
| go get -u golang.org/x/lint/golint | |
| go install google.golang.org/protobuf/cmd/[email protected] | |
| go install google.golang.org/grpc/cmd/[email protected] | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('*_python/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Set up Python | |
| run: | | |
| pip install wheel pytest | |
| pip install -e ./fleetspeak_python[test] | |
| pip install -e ./frr_python | |
| - name: Lint | |
| # We want to address all golint warnings, except for | |
| # https://github.com/golang/go/wiki/CodeReviewComments#doc-comments | |
| # TODO(mbushkov): make golint and go vet checks actionable. | |
| run: | | |
| golint ./... | grep -v 'should have comment or be unexported' || true | |
| go vet ./... || true | |
| - name: Check generated protos | |
| run: | | |
| fleetspeak/generate_go_py_protos.sh | |
| if [[ "$(git status --porcelain | grep .pb.go)" != "" ]]; then | |
| echo "At least one generated proto file is not in sync with the committed generated proto files." | |
| echo "Please run \`PATH=~/go/bin:\$PATH fleetspeak/generate_go_py_protos.sh\`." | |
| echo "pip packages:" | |
| pip freeze | |
| echo "git diff:" | |
| git diff | |
| exit 1; | |
| fi; | |
| echo "git status is clean; generated protos are consistent" | |
| - name: Build | |
| run: | | |
| CGO_ENABLED=0 go build ./cmd/... | |
| - name: Test | |
| run: | | |
| fleetspeak/test.sh | |
| - name: Check DEB installation | |
| # Install the built package and check that the fleetspeak-config program | |
| # doesn't error out. | |
| timeout-minutes: 10 | |
| run: | | |
| mysql -e 'DROP DATABASE ${{ env.MYSQL_TEST_E2E_DB }}; CREATE DATABASE ${{ env.MYSQL_TEST_E2E_DB }};' -u${{ env.MYSQL_TEST_USER }} -p${{ env.MYSQL_TEST_PASS }} | |
| sudo apt-get update | |
| sudo apt install debhelper devscripts fakeroot libparse-debcontrol-perl | |
| cd fleetspeak | |
| ./build-pkgs.sh | |
| # Pass through MySQL config environment variables to sudo. | |
| sudo -E ./test-package.sh ./fleetspeak-server_$(cat ../VERSION)_amd64.deb ./fleetspeak-client_$(cat ../VERSION)_amd64.deb | |
| - name: Build installers | |
| run: | | |
| DEPLOY_PATH=$GITHUB_WORKSPACE/deploy/ | |
| mkdir -p $DEPLOY_PATH | |
| cd fleetspeak | |
| cp ./fleetspeak-client_$(cat ../VERSION)_amd64.deb $DEPLOY_PATH | |
| cp ./fleetspeak-server_$(cat ../VERSION)_amd64.deb $DEPLOY_PATH | |
| # pypi doesn't support linux_x86_64, which is the plafrom name targeted by default. | |
| # We generate manylinux1_x86_64 manylinux2010_x86_64 packages, which are supported. | |
| # Create client wheel | |
| dpkg --extract ./fleetspeak-client_$(cat ../VERSION)_amd64.deb client-package-root | |
| python client-wheel/setup.py --package-root=client-package-root --version=$(cat ../VERSION) -- bdist_wheel --platform-name=manylinux1_x86_64 | |
| python client-wheel/setup.py --package-root=client-package-root --version=$(cat ../VERSION) -- bdist_wheel --platform-name=manylinux2010_x86_64 | |
| # Create server wheel | |
| dpkg --extract ./fleetspeak-server_$(cat ../VERSION)_amd64.deb server-package-root | |
| python server-wheel/setup.py --package-root=server-package-root --version=$(cat ../VERSION) -- bdist_wheel --platform-name=manylinux1_x86_64 | |
| python server-wheel/setup.py --package-root=server-package-root --version=$(cat ../VERSION) -- bdist_wheel --platform-name=manylinux2010_x86_64 | |
| # Copy wheels | |
| cp dist/*.whl $DEPLOY_PATH | |
| - if: ${{ github.event_name == 'push' }} | |
| name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux-installers | |
| path: deploy/ | |
| retention-days: 1 | |
| build-windows: | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.22' | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| powershell Install-WindowsFeature Net-Framework-Core | |
| pip install wheel | |
| - name: Build installers | |
| shell: bash | |
| run: | | |
| go build -v -x -o fleetspeak-client.exe ./cmd/fleetspeak_client/fleetspeak_client.go | |
| cd fleetspeak/client-win | |
| powershell -ExecutionPolicy Bypass -File ./build.ps1 | |
| cd ../.. | |
| DEPLOY_PATH=$GITHUB_WORKSPACE/deploy/ | |
| mkdir -p $DEPLOY_PATH | |
| cp ${TMP}/fleetspeak-build-*/fleetspeak-pkg/fleetspeak-client-*.msi $DEPLOY_PATH | |
| # Build client wheel | |
| mkdir pkg-root | |
| cp fleetspeak-client.exe pkg-root | |
| cp fleetspeak/client-win/fleetspeak_lib.wxs pkg-root | |
| python fleetspeak/client-wheel/setup.py --package-root pkg-root --version=$(cat VERSION) bdist_wheel | |
| cp dist/*.whl $DEPLOY_PATH | |
| ls -la $DEPLOY_PATH | |
| - if: ${{ github.event_name == 'push' }} | |
| name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: windows-installers | |
| path: deploy/ | |
| retention-days: 1 | |
| build-osx: | |
| runs-on: macos-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.22' | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| pip install wheel | |
| - name: Build installers | |
| run: | | |
| go build -o fleetspeak-client ./cmd/fleetspeak_client/fleetspeak_client.go | |
| DEPLOY_PATH=$GITHUB_WORKSPACE/deploy/ | |
| mkdir -p $DEPLOY_PATH | |
| cd fleetspeak/client-mac | |
| ./build.sh ../../fleetspeak-client | |
| sudo installer -pkg ./work/fleetspeak-client-*.pkg -target / -verbose | |
| cd ../.. | |
| cp ./fleetspeak/client-mac/work/fleetspeak-client-*.pkg $DEPLOY_PATH | |
| # Build client wheel | |
| python fleetspeak/client-wheel/setup.py --package-root=fleetspeak/client-mac/work/pkg_root --version=$(cat VERSION) bdist_wheel | |
| cp dist/*.whl $DEPLOY_PATH | |
| ls -la $DEPLOY_PATH | |
| - if: ${{ github.event_name == 'push' }} | |
| name: Upload installers to GitHub artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: osx-installers | |
| path: deploy/ | |
| retention-days: 1 | |
| test-osx: | |
| runs-on: macos-latest | |
| env: | |
| MYSQL_TEST_USER: root | |
| MYSQL_TEST_ADDR: 127.0.0.1:3306 | |
| MYSQL_TEST_E2E_DB: e2e_test_db | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.22' | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| pip install wheel pytest | |
| - name: Build Python | |
| run: | | |
| pip install -e ./fleetspeak_python[test] | |
| pip install -e ./frr_python | |
| - name: Build Go | |
| run: | | |
| CGO_ENABLED=0 go build ./cmd/... | |
| - uses: ankane/setup-mysql@v1 | |
| with: | |
| database: ${{ env.MYSQL_TEST_E2E_DB }} | |
| - name: Test | |
| run: | | |
| cd fleetspeak | |
| ./test.sh | |
| test-windows: | |
| runs-on: windows-latest | |
| defaults: | |
| run: | |
| working-directory: ${{ github.workspace }}/src/github.com/google/fleetspeak | |
| steps: | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.22' | |
| - uses: actions/checkout@v3 | |
| with: | |
| path: ${{ github.workspace }}/src/github.com/google/fleetspeak | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| # TODO: The dependency installation fails, but good enough to make the tests pass. | |
| # Ideally, required dependencies should be installed in a non-hacky, proper way. | |
| # go get -v -t ./... || echo "Dependency installation failed, continuing anyway ¯\_(ツ)_/¯" | |
| pip install wheel | |
| - name: Build | |
| shell: bash | |
| run: | | |
| go build -o fleetspeak/src/client/socketservice/testclient/testclient.exe github.com/google/fleetspeak/fleetspeak/src/client/socketservice/testclient | |
| go build -o fleetspeak/src/client/daemonservice/testclient/testclient.exe github.com/google/fleetspeak/fleetspeak/src/client/daemonservice/testclient | |
| pip install -e ./fleetspeak_python | |
| - name: Test | |
| shell: bash | |
| run: | | |
| go test -race github.com/google/fleetspeak/fleetspeak/src/common/... --timeout 180s | |
| go test -race github.com/google/fleetspeak/fleetspeak/src/client/... --timeout 180s | |
| # TODO: Move src/windows to src/client. | |
| go test -race github.com/google/fleetspeak/fleetspeak/src/windows/... --timeout 180s | |
| upload-installers: | |
| if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} | |
| runs-on: ubuntu-latest | |
| env: | |
| GCS_BUCKET: autobuilds-fleetspeak | |
| needs: | |
| - build-test-linux | |
| - build-windows | |
| - build-osx | |
| - test-osx | |
| - test-windows | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download installers from GitHub artifacts | |
| id: download | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ~/_artifacts | |
| - name: Merge artifacts | |
| run: | | |
| BUILD_TIMESTAMP=$(git show -s --format=%ci ${GITHUB_SHA} | sed -e "s/[ :\\+]/_/g") | |
| DEPLOY_PATH=$HOME/deploy/${BUILD_TIMESTAMP}_${GITHUB_SHA}/ | |
| echo "DEPLOY_PATH=$DEPLOY_PATH" >> $GITHUB_ENV | |
| mkdir -p $DEPLOY_PATH | |
| ls -la ${{ steps.download.outputs.download-path }}/* | |
| mv -v ${{ steps.download.outputs.download-path }}/*/* $DEPLOY_PATH | |
| ls -la $DEPLOY_PATH | |
| echo "BUILD_TIMESTAMP=${BUILD_TIMESTAMP}" >> $GITHUB_ENV | |
| - name: Authenticate | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| export_environment_variables: true | |
| - name: Upload installers to GCS | |
| uses: google-github-actions/[email protected] | |
| with: | |
| path: ${{ env.DEPLOY_PATH }} | |
| destination: ${{ env.GCS_BUCKET }}/${{ env.BUILD_TIMESTAMP }}_${{ github.sha }} | |
| # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. | |
| parent: false | |
| build-push-docker: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Login to GitHub Container registry | |
| if: ${{ github.event_name == 'push' }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| if: ${{ github.event_name == 'push' }} | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| - name: Build and push Docker image | |
| if: ${{ github.event_name == 'push' }} | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} |