ci: updated workflow #625
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Docker Image CI: crawlab" | |
| on: | |
| push: | |
| branches: [ develop, test, main ] | |
| release: | |
| types: [ published ] | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [ docker-crawlab ] | |
| env: | |
| IMAGE_PATH_CRAWLAB_BACKEND: backend | |
| IMAGE_PATH_CRAWLAB_FRONTEND: frontend | |
| GH_PKG_NAME_CRAWLAB_BASE: ${{ github.repository_owner }}/crawlab-base | |
| GH_PKG_NAME_CRAWLAB_BACKEND: ${{ github.repository_owner }}/crawlab-backend | |
| GH_PKG_NAME_CRAWLAB_FRONTEND: ${{ github.repository_owner }}/crawlab-frontend | |
| GH_PKG_NAME_CRAWLAB: ${{ github.repository_owner }}/crawlab | |
| IMAGE_NAME_CRAWLAB_BASE: ghcr.io/${{ github.repository_owner }}/crawlab-base | |
| IMAGE_NAME_CRAWLAB_BACKEND: ghcr.io/${{ github.repository_owner }}/crawlab-backend | |
| IMAGE_NAME_CRAWLAB_FRONTEND: ghcr.io/${{ github.repository_owner }}/crawlab-frontend | |
| IMAGE_NAME_CRAWLAB: ghcr.io/${{ github.repository_owner }}/crawlab | |
| IMAGE_NAME_CRAWLAB_DOCKERHUB: crawlabteam/crawlab | |
| IMAGE_NAME_CRAWLAB_TENCENT: ccr.ccs.tencentyun.com/crawlab/crawlab | |
| E2E_TESTS_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/e2e-tests | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend_changed: ${{ steps.check_changed_files.outputs.backend_changed }} | |
| frontend_changed: ${{ steps.check_changed_files.outputs.frontend_changed }} | |
| dockerfile_changed: ${{ steps.check_changed_files.outputs.dockerfile_changed }} | |
| workflow_changed: ${{ steps.check_changed_files.outputs.workflow_changed }} | |
| base_image_changed: ${{ steps.check_changed_files.outputs.base_image_changed }} | |
| version: ${{ steps.version.outputs.version }} | |
| test_script: ${{ steps.test_config.outputs.test_script }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get changed files | |
| id: changed_files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| files_yaml: | | |
| backend: | |
| - 'backend/**' | |
| - 'core/**' | |
| - 'db/**' | |
| - 'fs/**' | |
| - 'grpc/**' | |
| - 'vcs/**' | |
| - 'trace/**' | |
| frontend: | |
| - '${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }}/**' | |
| dockerfile: | |
| - 'Dockerfile' | |
| workflow: | |
| - '.github/workflows/docker-crawlab.yml' | |
| base_image: | |
| - 'docker/base-image/**' | |
| - id: check_changed_files | |
| name: Check changed files | |
| run: | | |
| # set outputs | |
| if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then | |
| echo "backend_changed=true" >> $GITHUB_OUTPUT | |
| echo "frontend_changed=true" >> $GITHUB_OUTPUT | |
| echo "dockerfile_changed=true" >> $GITHUB_OUTPUT | |
| echo "workflow_changed=true" >> $GITHUB_OUTPUT | |
| echo "base_image_changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "backend_changed=${{ steps.changed_files.outputs.backend_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "frontend_changed=${{ steps.changed_files.outputs.frontend_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "dockerfile_changed=${{ steps.changed_files.outputs.dockerfile_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "workflow_changed=${{ steps.changed_files.outputs.workflow_any_changed }}" >> $GITHUB_OUTPUT | |
| echo "base_image_changed=${{ steps.changed_files.outputs.base_image_any_changed }}" >> $GITHUB_OUTPUT | |
| fi | |
| # Display change status for each component | |
| echo "Backend changed: ${{ steps.changed_files.outputs.backend_any_changed }}" | |
| echo "Frontend changed: ${{ steps.changed_files.outputs.frontend_any_changed }}" | |
| echo "Dockerfile changed: ${{ steps.changed_files.outputs.dockerfile_any_changed }}" | |
| echo "Workflow changed: ${{ steps.changed_files.outputs.workflow_any_changed }}" | |
| echo "Base image changed: ${{ steps.changed_files.outputs.base_image_any_changed }}" | |
| - id: version | |
| name: Get version | |
| run: | | |
| # Strip git ref prefix from version | |
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | |
| # Strip "v" prefix from tag name | |
| [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') | |
| # Use Docker `latest` tag convention | |
| [ "$VERSION" == "main" ] && VERSION=latest | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| - id: test_config | |
| name: Set test configuration | |
| run: | | |
| if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "test_script=test:full" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.ref }}" == "refs/heads/test" ]]; then | |
| echo "test_script=test:extended" >> $GITHUB_OUTPUT | |
| else | |
| echo "test_script=test:normal" >> $GITHUB_OUTPUT | |
| fi | |
| build_base_image: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.base_image_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ./docker/base-image | |
| push: true | |
| tags: ${{ env.IMAGE_NAME_CRAWLAB_BASE }}:${{ needs.setup.outputs.version }} | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| test_backend: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.backend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| services: | |
| mongo: | |
| image: mongo:5 | |
| ports: | |
| - 27017:27017 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'core/go.mod' | |
| cache-dependency-path: 'core/go.sum' | |
| - name: Run tests | |
| working-directory: core | |
| run: | | |
| # Find all directories containing *_test.go files | |
| test_dirs=$(find . -name "*_test.go" -exec dirname {} \; | sort -u) | |
| # Run go test on each directory | |
| for dir in $test_dirs | |
| do | |
| echo "Running tests in $dir" | |
| go test ./$dir | |
| done | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| build_backend: | |
| needs: [ setup, test_backend ] | |
| if: needs.test_backend.result == 'success' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ env.IMAGE_PATH_CRAWLAB_BACKEND }} | |
| push: true | |
| tags: ${{ env.IMAGE_NAME_CRAWLAB_BACKEND }}:${{ needs.setup.outputs.version }} | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| build_frontend: | |
| needs: [ setup ] | |
| if: needs.setup.outputs.frontend_changed == 'true' || needs.setup.outputs.workflow_changed == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| failed: ${{ steps.set_output.outputs.failed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: ${{ env.IMAGE_PATH_CRAWLAB_FRONTEND }} | |
| push: true | |
| tags: ${{ env.IMAGE_NAME_CRAWLAB_FRONTEND }}:${{ needs.setup.outputs.version }} | |
| - name: Set output | |
| id: set_output | |
| if: failure() | |
| run: echo "failed=true" >> $GITHUB_OUTPUT | |
| build_crawlab: | |
| needs: [setup, build_base_image, test_backend, build_backend, build_frontend] | |
| if: | | |
| always() && | |
| ( | |
| needs.test_backend.outputs.failed != 'true' && | |
| needs.build_backend.outputs.failed != 'true' && | |
| needs.build_frontend.outputs.failed != 'true' && | |
| needs.build_base_image.outputs.failed != 'true' | |
| ) && | |
| ( | |
| needs.setup.outputs.backend_changed == 'true' || | |
| needs.setup.outputs.frontend_changed == 'true' || | |
| needs.setup.outputs.dockerfile_changed == 'true' || | |
| needs.setup.outputs.base_image_changed == 'true' || | |
| needs.setup.outputs.workflow_changed == 'true' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Update Dockerfile | |
| run: | | |
| IMAGE_NAMES=( | |
| "crawlab-base" | |
| "crawlab-backend" | |
| "crawlab-frontend" | |
| ) | |
| for name in "${IMAGE_NAMES[@]}"; do | |
| IMAGE_NAME="ghcr.io/${{ github.repository_owner }}/$name" | |
| OLD_IMAGE="crawlabteam/${name}:" | |
| NEW_IMAGE="${IMAGE_NAME}:" | |
| sed -i "s|${OLD_IMAGE}|${NEW_IMAGE}|" Dockerfile | |
| done | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./Dockerfile | |
| build-args: | | |
| CRAWLAB_TAG=${{ needs.setup.outputs.version }} | |
| push: true | |
| tags: | | |
| ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} | |
| test_crawlab: | |
| needs: [setup, build_crawlab] | |
| if: ${{ always() && needs.build_crawlab.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| services: | |
| mongo: | |
| image: mongo:5 | |
| options: >- | |
| --health-cmd "mongosh --eval 'db.adminCommand(\"ping\")' || exit 1" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| master: | |
| image: ghcr.io/${{ github.repository_owner }}/crawlab:${{ needs.setup.outputs.version }} | |
| env: | |
| CRAWLAB_NODE_MASTER: Y | |
| CRAWLAB_MONGO_HOST: mongo | |
| CRAWLAB_MONGO_PORT: 27017 | |
| ports: | |
| - 8080:8080 | |
| options: >- | |
| --health-cmd "curl -f http://localhost:8080/api/system-info || exit 1" | |
| --health-interval 30s | |
| --health-timeout 10s | |
| --health-retries 5 | |
| worker: | |
| image: ghcr.io/${{ github.repository_owner }}/crawlab:${{ needs.setup.outputs.version }} | |
| env: | |
| CRAWLAB_NODE_MASTER: N | |
| CRAWLAB_MASTER_HOST: master | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull e2e test image | |
| run: | | |
| docker pull ${{ env.E2E_TESTS_IMAGE_NAME }}:latest | |
| - name: Run e2e test image | |
| run: | | |
| docker run --network host \ | |
| -e BASE_URL=http://localhost:8080 \ | |
| -e TEST_SCRIPT=${{ needs.setup.outputs.test_script }} \ | |
| -v ${{ github.workspace }}/playwright-report:/app/playwright-report \ | |
| ${{ env.E2E_TESTS_IMAGE_NAME }}:latest | |
| - name: Deploy to GitHub Pages | |
| if: always() | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ${{ github.workspace }}/playwright-report | |
| destination_dir: playwright-report/${{ needs.setup.outputs.version }} | |
| push_images: | |
| if: ${{ always() && needs.test_crawlab.result == 'success' }} | |
| needs: [setup, test_crawlab] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| registry: [dockerhub, tencent] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pull Docker image from GitHub Container Registry | |
| run: docker pull ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} | |
| - name: Login to DockerHub or Tencent Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ (matrix.registry == 'dockerhub' && 'https://index.docker.io/v1/') || (matrix.registry == 'tencent' && 'ccr.ccs.tencentyun.com') }} | |
| username: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_USERNAME) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_USERNAME) }} | |
| password: ${{ (matrix.registry == 'dockerhub' && secrets.DOCKER_PASSWORD) || (matrix.registry == 'tencent' && secrets.DOCKER_TENCENT_PASSWORD) }} | |
| - name: Tag and push image | |
| if: ${{ matrix.registry == 'dockerhub' || (matrix.registry == 'tencent' && github.ref != 'refs/heads/develop') }} | |
| run: | | |
| docker tag ${{ env.IMAGE_NAME_CRAWLAB }}:${{ needs.setup.outputs.version }} ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB_DOCKERHUB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} | |
| docker push ${{ (matrix.registry == 'dockerhub' && env.IMAGE_NAME_CRAWLAB_DOCKERHUB) || (matrix.registry == 'tencent' && env.IMAGE_NAME_CRAWLAB_TENCENT) }}:${{ needs.setup.outputs.version }} |