aws_runner.yml

name: aws_runner
on: workflow_dispatch

jobs:
  start-runner:
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }}
          aws-region: ${{ vars.AWS_REGION }}
      - name: Start EC2 runner
        id: start-ec2-runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: start
          github-token: ${{ secrets.GH_TOKEN }}
          ec2-image-id: ${{ vars.AWS_IMAGE_ID }}
          ec2-instance-type: ${{ vars.AWS_INSTANCE_TYPE }}
          subnet-id: ${{ vars.AWS_SUBNET }}
          security-group-id: ${{ vars.AWS_SECURITY_GROUP }}

  test-model:
    name: Do the job on the runner
    needs: [start-runner] # push-model, build, build-docker, push-weights, start-runner] # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
    steps:
      - run: export HOME=/home/ubuntu
      - name: do it
        run: echo $HOME
        # Setups singularity to the job to make it accessible to other steps
        # Cleanup steps to free up disk space
      - name: Cleanup disk space for large docker images
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf /opt/hostedtoolcache
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
            
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          # this might remove tools that are actually needed,
          # if set to "true" but frees about 6 GB
          tool-cache: false
          
          # all of these default to true, but feel free to set to
          # "false" if necessary for your workflow
          android: true
          dotnet: true
          haskell: true
          large-packages: true
          docker-images: false
      
      # Datalad get the sample dataset
      - name: Get sample dataset
        run: |
          apt install unzip datalad python3-pip -y
          python3 -m pip install datalad-installer datalad-osf
          datalad-installer --sudo ok git-annex -m datalad/git-annex:release
          sudo git config --global filter.annex.process "git-annex filter-process"
      - name: configure git credentials
        run: |
          git config --system user.name "test_aws"
          git config --system user.email "test_aws"
      # If branchName is empty use issue number, else, use the branchName 🟢
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: datalad get
        run: |
          datalad siblings
          datalad siblings
          cd PialNN/pialnn/1.0.0/data
          datalad get .
          find . -name "*.zip" -exec unzip {} \;
          mv example example_aws2
      - name: datalad push
        run: |
          datalad save -m "pushing new unzipped folder" PialNN/pialnn/1.0.0/data/example_aws2
          datalad push -f all --to test-aws-storage
          datalad push --to origin
        env:
          OSF_TOKEN: ${{ secrets.OSF_TOKEN }}
  
  stop-runner:
    name: Stop self-hosted EC2 runner
    needs: [start-runner, test-model] # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }}
    # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
            aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
            aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }}
            aws-region: ${{ vars.AWS_REGION }}
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.GH_TOKEN }}
          label: ${{ needs.start-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}