aws_runner #23
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: aws_runner | |
| on: workflow_dispatch | |
| jobs: | |
| start-runner: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_TOKEN }} | |
| ec2-image-id: ${{ vars.AWS_IMAGE_ID }} | |
| ec2-instance-type: ${{ vars.AWS_INSTANCE_TYPE }} | |
| subnet-id: ${{ vars.AWS_SUBNET }} | |
| security-group-id: ${{ vars.AWS_SECURITY_GROUP }} | |
| test-model: | |
| name: Do the job on the runner | |
| needs: [start-runner] # push-model, build, build-docker, push-weights, start-runner] # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner | |
| steps: | |
| - run: export HOME=/home/ubuntu | |
| - name: do it | |
| run: echo $HOME | |
| # Setups singularity to the job to make it accessible to other steps | |
| # Cleanup steps to free up disk space | |
| - name: Cleanup disk space for large docker images | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf /opt/hostedtoolcache | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| # this might remove tools that are actually needed, | |
| # if set to "true" but frees about 6 GB | |
| tool-cache: false | |
| # all of these default to true, but feel free to set to | |
| # "false" if necessary for your workflow | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: false | |
| # Datalad get the sample dataset | |
| - name: Get sample dataset | |
| run: | | |
| apt-get update | |
| apt install unzip datalad python3-pip tree -y | |
| python3 -m pip install datalad-installer datalad-osf | |
| datalad-installer --sudo ok git-annex -m datalad/git-annex:release | |
| sudo git config --global filter.annex.process "git-annex filter-process" | |
| # - name: configure git credentials | |
| # run: | | |
| # git config --system user.name "test_aws" | |
| # git config --system user.email "test_aws" | |
| - name: Set up Git | |
| run: | | |
| export HOME=/home/ubuntu | |
| echo $HOME | |
| git config --global user.email "test_aws" | |
| git config --global user.name "test_aws" | |
| git config --global credential.helper cache | |
| git config --global credential.https://github.com.username "${{ secrets.GITHUB_USERNAME }}" | |
| git config --global credential.https://github.com.password "${{ secrets.GITHUB_TOKEN }}" | |
| env: | |
| GITHUB_USERNAME: ${{ secrets.GH_USERNAME }} | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} | |
| # If branchName is empty use issue number, else, use the branchName 🟢 | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: datalad get | |
| run: | | |
| datalad siblings | |
| datalad siblings | |
| cd PialNN/pialnn/1.0.0/data | |
| datalad get . | |
| find . -name "*.zip" -exec unzip {} \; | |
| mv example example_aws4 | |
| - run: | | |
| git config user.name | |
| git config user.email | |
| datalad-push: | |
| name: running datalad push as a separate job | |
| needs: [start-runner, test-model] | |
| runs-on: ${{ needs.start-runner.outputs.label }} | |
| steps: | |
| - run: | | |
| git config user.name | |
| git config user.email | |
| - run: tree | |
| - name: datalad push | |
| run: | | |
| datalad save -m "pushing new unzipped folder" PialNN/pialnn/1.0.0/data/example_aws4 | |
| tree | |
| datalad push -f all --to test-aws-storage | |
| datalad push --to origin | |
| env: | |
| OSF_TOKEN: ${{ secrets.OSF_TOKEN }} | |
| stop-runner: | |
| name: Stop self-hosted EC2 runner | |
| needs: [start-runner, test-model, datalad-push] # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| # required to stop the runner even if the error happened in the previous jobs | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} |