forked from bioimage-io/collection-bioimage-io
-
Notifications
You must be signed in to change notification settings - Fork 0
258 lines (239 loc) · 10.2 KB
/
auto_update_main.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
name: update collection
concurrency: ${{ github.ref }} # do not run this workflow in parallel for the same branch (in particular main)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
schedule:
- cron: '0 3 * * *'
workflow_dispatch: # triggered by http endpoints (ref should be refs/head/main)
env:
BIOIMAGEIO_COUNT_RDF_DOWNLOADS: 'false'
jobs:
clean-up-pr:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- name: delete bot comments
continue-on-error: true
shell: python
run: |
import requests
r = requests.get(
"${{ github.api_url }}/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments",
headers=dict(Accept="application/vnd.github.v3+json"),
)
r.raise_for_status()
bot_comment_ids = [c["id"] for c in r.json() if c.get("user", {}).get("login") == "bioimageiobot"]
for comment_id in bot_comment_ids:
r = requests.delete(
f"${{ github.api_url }}/repos/${{ github.repository }}/issues/comments/{comment_id}",
headers=dict(Accept="application/vnd.github.v3+json", Authorization="token ${{ secrets.PAT }}"),
)
r.raise_for_status()
update-resources:
if: ${{ !startsWith(github.head_ref || github.ref, 'refs/heads/auto-update-') }}
runs-on: ubuntu-latest
outputs:
updated_resources_matrix: ${{ steps.update_external.outputs.updated_resources_matrix }}
found_new_resources: ${{ steps.update_external.outputs.found_new_resources || 'no' }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # get all branches to check for pending resources by checking if auto-update-... branch exists
- name: checkout gh-pages for update_partner_resources.py
uses: actions/checkout@v3
with:
ref: gh-pages
path: gh-pages
- name: install script deps
uses: mamba-org/setup-micromamba@v1
with:
cache-downloads: true
cache-environment: true
environment-name: scriptenv
condarc: |
channels:
- conda-forge
create-args: >- # script dependencies
bioimageio.spec
bs4
lxml
requests
typer
- name: update external resources
id: update_external
shell: bash -l {0}
run: python scripts/update_external_resources.py --max-resource-count 20 ${{ github.event_name == 'pull_request' && '--no-ignore-status-5xx' || '--ignore-status-5xx' }}
- name: "Upload collection update"
uses: actions/upload-artifact@v3
with:
name: collection-update
path: dist
retention-days: 30
- name: clean up dist
run: |
mv dist/download_counts.json tmp_download_counts.json
rm -r dist
mkdir dist
mv tmp_download_counts.json dist/download_counts.json
- name: update partner resources
shell: bash -l {0}
run: python scripts/update_partner_resources.py
- name: Upload preview of updated partner resources
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@v3
with:
name: preview-partner-resources
path: dist
retention-days: 90
- name: Deploy updated partner resources to gh-pages 🚀
if: github.event_name != 'pull_request'
uses: JamesIves/[email protected]
with:
clean: false
branch: gh-pages
folder: dist
validate:
needs: update-resources
uses: fynnbe/collection-bioimage-io/.github/workflows/validate_resources.yaml@main # todo: change orga to bioimage-io
with:
check_validation: ${{ github.event_name == 'pull_request' && 'yes' || 'no' }}
deploy_to: ${{ github.event_name == 'pull_request' && 'preview' || 'gh-pages' }}
trigger-partner-validation:
needs: validate
if: github.event_name != 'pull_request' && needs.validate.outputs.has_pending_matrices == 'yes'
uses: fynnbe/collection-bioimage-io/.github/workflows/trigger_partner_wf.yaml@main # todo: change orga to bioimage-io
with:
pending_matrices: ${{ needs.validate.outputs.pending_matrices }}
secrets:
PAT: ${{ secrets.PAT }}
build-collection: # deploy collection.json
needs: validate
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
with:
ref: gh-pages
path: gh-pages
- name: install script deps
uses: mamba-org/setup-micromamba@v1
with:
cache-downloads: true
cache-environment: true
environment-name: scriptenv
condarc: |
channels:
- conda-forge
create-args: >- # script dependencies
bioimageio.spec
boltons
lxml
requests
typer
pooch
pillow
- name: generate collection rdf and thumbnails
shell: bash -l {0}
run: python scripts/generate_collection_rdf_and_thumbnails.py
- name: Upload preview of collection.json
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@v3
with:
name: preview-collection-json
path: dist/collection.json
retention-days: 90
- name: Deploy collection.json to gh-pages 🚀
if: github.event_name != 'pull_request'
uses: JamesIves/[email protected]
with:
clean: false
branch: gh-pages
folder: dist
- name: update last_ci_run tag
if: github.event_name != 'pull_request'
uses: octokit/[email protected]
with:
route: POST /repos/${{ github.repository }}/git/refs/tags/last_ci_run
sha: ${{ github.sha }}
env:
GITHUB_TOKEN: ${{ secrets.PAT }}
open-pr:
needs: [update-resources, build-collection] # run after build-collection to not deploy to gh-pages in parallel
if: ${{ (!startsWith(github.head_ref || github.ref, 'refs/heads/auto-update-')) && needs.update-resources.outputs.found_new_resources == 'yes' }}
runs-on: ubuntu-latest
strategy:
max-parallel: 1 # to limit content creation (PRs) on GitHub to avoid getting temporary blocked
fail-fast: false
# matrix: update: [{resource_id:..., resource_name:..., new_version_ids:..., new_version_sources:..., maintainers: ...}, ...]
matrix: ${{ fromJson(needs.update-resources.outputs.updated_resources_matrix) }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3
with:
ref: gh-pages
path: gh-pages
- run: mkdir dist
- run: cp gh-pages/pr_urls.json dist
continue-on-error: true
- name: download new collection
uses: actions/download-artifact@v3
with:
name: collection-update
path: collection-update
- name: replace resource item folder
run: |
mkdir -p "collection/${{ matrix.update.resource_id }}"
cp -r "collection-update/${{ matrix.update.resource_id }}"/* "collection/${{ matrix.update.resource_id }}"
rm -rf collection-update
- name: get urls of previous PRs
id: pr_urls
run: python scripts/get_previous_pr_urls.py ${{ matrix.update.resource_id }}
- name: Create Pull Request
if: github.event_name != 'pull_request'
id: cpr
uses: peter-evans/create-pull-request@v4
with:
token: ${{ secrets.PAT }} # using a PAT will make sure we can trigger the CI in PR
commit-message: 'Update ${{ matrix.update.resource_id }}'
committer: GitHub <[email protected]>
author: bioimage.io Bot <[email protected]>
signoff: false
branch: auto-update-${{ matrix.update.resource_id }}
delete-branch: false # true does not seem to actually delete the PR branch...? we delete the branch in block_pending.yaml
title: 'Update ${{ matrix.update.resource_name }}'
body: |
This is an automatic PR created by the @bioimageiobot regarding changes to the resource item ${{ matrix.update.resource_id }}.
The following version(s) will be added:
${{ matrix.update.new_version_ids_md }}
Please review the changes and make sure the new item or version(s) pass the following check list:
- [ ] Passed the bioimage.io CI tests: static (and dynamic) validations
- [ ] The meta information for the RDF item is complete
- [ ] The tags are complete and describe the model
- [ ] Naming is intuitive and descriptive, example: Multi-Organ Nucleus Segmentation (StarDist 2D)
- [ ] Authors are provided
- [ ] Documentation is complete
* For models, include an overview, describe how the model is trained, what is the training data, how to use the model, how to validate the results and list the references. TODO: Model documentation template.
- [ ] Approved by at least one of the bioimage.io admin team member.
Maintainers: ${{ matrix.update.maintainers }}
Note: If you updated or re-uploaded another version for the current item on Zenodo, this PR won't be changed automatically. To proceed, you can do the following:
1. Block this version, but keep looking for future versions: Edit the current resource.yaml and keep the top-level `status` field as `accepted`, but change the `status` under the current version to `blocked`.
2. Accept this version and keep looking for future versions: Merge this PR for now.
3. Keep proposed version(s) (and this resource in general if it is new) as pending: Close this PR without merging.
Then wait for the CI on the main branch to complete. It should detect the new version(s) and create another PR for the new version(s).
Previous PRs of this resource: ${{ steps.pr_urls.outputs.previous_prs }}
labels: |
auto-update
draft: false
- name: save PR url
if: github.ref == 'refs/heads/main'
run: python scripts/save_pr_url.py ${{ matrix.update.resource_id }} ${{ steps.cpr.outputs.pull-request-url }}
- name: deploy PR url to gh-pages 🚀
if: github.event_name != 'pull_request'
uses: JamesIves/[email protected]
with:
clean: false
branch: gh-pages
folder: dist