-
Notifications
You must be signed in to change notification settings - Fork 0
111 lines (104 loc) · 4.25 KB
/
aws_runner.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
name: aws_runner
on: workflow_dispatch
jobs:
start-runner:
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }}
aws-region: ${{ vars.AWS_REGION }}
- name: Start EC2 runner
id: start-ec2-runner
uses: machulav/ec2-github-runner@v2
with:
mode: start
github-token: ${{ secrets.GH_TOKEN }}
ec2-image-id: ${{ vars.AWS_IMAGE_ID }}
ec2-instance-type: ${{ vars.AWS_INSTANCE_TYPE }}
subnet-id: ${{ vars.AWS_SUBNET }}
security-group-id: ${{ vars.AWS_SECURITY_GROUP }}
test-model:
name: Do the job on the runner
needs: [start-runner] # push-model, build, build-docker, push-weights, start-runner] # required to start the main job when the runner is ready
runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
steps:
- run: echo ${{ secrets.DOCKER_USERNAME }}
- run: pwd
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
# - run: export HOME=/home/ubuntu
# - name: do it
# run: echo $HOME
# # Setups singularity to the job to make it accessible to other steps
# # Cleanup steps to free up disk space
# - name: Cleanup disk space for large docker images
# run: |
# sudo rm -rf /usr/share/dotnet
# sudo rm -rf /opt/ghc
# sudo rm -rf "/usr/local/share/boost"
# sudo rm -rf /opt/hostedtoolcache
# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
# - name: Free Disk Space (Ubuntu)
# uses: jlumbroso/free-disk-space@main
# with:
# # this might remove tools that are actually needed,
# # if set to "true" but frees about 6 GB
# tool-cache: false
# # all of these default to true, but feel free to set to
# # "false" if necessary for your workflow
# android: true
# dotnet: true
# haskell: true
# large-packages: true
# docker-images: false
# # Datalad get the sample dataset
# - name: Get sample dataset
# run: |
# apt install unzip datalad python3-pip -y
# python3 -m pip install datalad-installer datalad-osf
# datalad-installer --sudo ok git-annex -m datalad/git-annex:release
# sudo git config --global filter.annex.process "git-annex filter-process"
# git config --system user.name "test_aws"
# git config --system user.email "test_aws"
# # If branchName is empty use issue number, else, use the branchName 🟢
# - uses: actions/checkout@v4
# with:
# fetch-depth: 0
# - run: |
# datalad siblings
# datalad siblings
# cd PialNN/pialnn/1.0.0/data
# datalad get .
# find . -name "*.zip" -exec unzip {} \;
# mv example example_aws
# datalad save -m "pushing unzipped folder" example_aws/
# datalad push -f all --to test-aws-storage
# datalad push --to origin
# env:
# OSF_TOKEN: ${{ secrets.OSF_TOKEN }}
stop-runner:
name: Stop self-hosted EC2 runner
needs: [start-runner, test-model] # required to wait when the main job is done
runs-on: ubuntu-latest
if: ${{ always() }}
# required to stop the runner even if the error happened in the previous jobs
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }}
aws-region: ${{ vars.AWS_REGION }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2
with:
mode: stop
github-token: ${{ secrets.GH_TOKEN }}
label: ${{ needs.start-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}