-
Notifications
You must be signed in to change notification settings - Fork 8
/
.gitlab-ci.yml
189 lines (182 loc) · 7.11 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
include:
- project: nse/ci
file:
- /ci/lib/common.yml
- project: cs/gitlabci-templates
file: /build-image-using-buildah.yml
workflow:
rules:
- if: '$CI_EXTERNAL_PULL_REQUEST_IID'
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
- if: $CI_PIPELINE_SOURCE == "web"
# Run workflows on tag creation to build the container
- if: $CI_COMMIT_TAG
variables:
BLUECONFIGS_BRANCH:
value: $BLUECONFIGS_BRANCH
description: 'Name of the blueconfigs branch to test against'
GIT_DEPTH: 100
set_alt_branches:
script:
- grep '^[[:alnum:]_]*_BRANCH=' <<< "$CI_COMMIT_MESSAGE" > branch_variables.env || touch branch_variables.env
- . branch_variables.env
- echo "BLUECONFIGS_PIPELINE_BRANCH=$BLUECONFIGS_BRANCH" >> branch_variables.env
- cat branch_variables.env
artifacts:
reports:
dotenv: branch_variables.env
blueconfig_tests:
variables:
bb5_build_dir: pipeline
PY_NEURODAMUS_BRANCH: $CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_NAME
PARENT_COMMIT_MESSAGE: $CI_COMMIT_MESSAGE
trigger:
project: hpc/sim/blueconfigs
branch: $BLUECONFIGS_PIPELINE_BRANCH
strategy: depend
needs:
- set_alt_branches
rules:
- if: '$CI_EXTERNAL_PULL_REQUEST_IID'
changes:
- neurodamus/**/*
- when: manual
allow_failure: true
configure-docker-image:
image: ubuntu:22.04
stage: build
script:
- set -x
- apt-get update -y
- apt-get install -y git skopeo python3
- |
echo "Getting package version and checking whether we need to build the container"
if [ -n "${LIBSONATAREPORT_TAG}" || -n "${LIBSONATA_TAG}" || -n "${NEURON_COMMIT_ID}" && -z "${REGISTRY_IMAGE_TAG}" ]; then
echo "If you specify LIBSONATAREPORT_TAG, LIBSONATA_TAG or NEURON_COMMIT_ID, you must also specify REGISTRY_IMAGE_TAG."
exit 1
fi
if [ -z "${REGISTRY_IMAGE_TAG}" ]; then
PACKAGE_VERSION=$(git describe --tags)
REGISTRY_IMAGE_TAG=$(echo ${PACKAGE_VERSION%-*} | sed 's/-/.dev/')
fi
set +e
skopeo inspect docker://bluebrain/neurodamus:${REGISTRY_IMAGE_TAG}
if [[ $? -eq 0 ]]; then
echo "Container with version ${REGISTRY_IMAGE_TAG} already exists - not rebuilding"
BUILD_DOCKER_CONTAINER="false"
else
echo "Container with version ${REGISTRY_IMAGE_TAG} does not exist yet - building"
BUILD_DOCKER_CONTAINER="true"
fi
set -e
- |
echo "Getting dependency versions from spack"
git clone https://github.com/bluebrain/spack
cd spack
source ./share/spack/setup-env.sh
if [ -z "${LIBSONATAREPORT_TAG}" ]; then
LIBSONATAREPORT_TAG=$(spack info libsonata-report | awk 'matched{print $NF;matched=0} /Preferred/{matched=1}')
fi
if [ -z "${LIBSONATA_TAG}" ]; then
LIBSONATA_TAG=v$(spack info py-libsonata | awk 'matched{print $1;matched=0} /Preferred/{matched=1}')
fi
if [ -z "${NEURON_COMMIT_ID}" ]; then
NEURON_COMMIT_ID=$(spack info neuron | awk 'matched{print $NF;matched=0} /Preferred/{matched=1}')
fi
cd ..
- |
cat <<EOF > build.env
REGISTRY_IMAGE_TAG=$REGISTRY_IMAGE_TAG
BUILD_DOCKER_CONTAINER=$BUILD_DOCKER_CONTAINER
LIBSONATAREPORT_TAG=${LIBSONATAREPORT_TAG}
LIBSONATA_TAG=${LIBSONATA_TAG}
NEURON_COMMIT_ID=${NEURON_COMMIT_ID}
EOF
- cat build.env
artifacts:
when: always
paths: [build.env]
reports:
dotenv: build.env
rules:
- if: $CI_COMMIT_TAG
when: manual
allow_failure: true
- if: $CI_PIPELINE_SOURCE == "web"
build-docker-image:
timeout: 2h
needs: [configure-docker-image]
stage: build
extends: .build-image-using-buildah
variables:
BUILD_PATH: $CI_PROJECT_DIR/docker
KUBERNETES_CPU_LIMIT: 8
KUBERNETES_CPU_REQUEST: 8
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_MEMORY_REQUEST: 8Gi
BUILDAH_EXTRA_ARGS:
--label org.opencontainers.image.title="neurodamus"
--label org.opencontainers.image.version="$REGISTRY_IMAGE_TAG"
--label org.opencontainers.image.revision="$CI_COMMIT_SHA"
--label org.opencontainers.image.authors="$GITLAB_USER_NAME <$GITLAB_USER_EMAIL>"
--label org.opencontainers.image.url="$CI_PROJECT_URL"
--label org.opencontainers.image.source="$CI_PROJECT_URL"
--label org.opencontainers.image.created="$CI_JOB_STARTED_AT"
--label ch.epfl.bbpgitlab.ci-pipeline-url="$CI_PIPELINE_URL"
--label ch.epfl.bbpgitlab.ci-commit-branch="$CI_COMMIT_BRANCH"
--label ch.epfl.bbpgitlab.libsonata-report-tag="$LIBSONATAREPORT_TAG"
--label ch.epfl.bbpgitlab.libsonata-tag="$LIBSONATA_TAG"
--label ch.epfl.bbpgitlab.neuron-commit-id="$NEURON_COMMIT_ID"
--build-arg GITLAB_CI="$GITLAB_CI"
--build-arg CI_JOB_TOKEN="$CI_JOB_TOKEN"
--build-arg LIBSONATAREPORT_TAG="$LIBSONATAREPORT_TAG"
--build-arg LIBSONATA_TAG="$LIBSONATA_TAG"
--build-arg NEURON_COMMIT_ID="$NEURON_COMMIT_ID"
before_script:
- echo "LIBSONATAREPORT_TAG is .$LIBSONATAREPORT_TAG."
- echo "LIBSONATA_TAG is .$LIBSONATA_TAG."
- echo "NEURON_COMMIT_ID is .$NEURON_COMMIT_ID."
- echo "REGISTRY_IMAGE_TAG is .$REGISTRY_IMAGE_TAG."
- set -x
- if [ "$BUILD_DOCKER_CONTAINER" == "false" ]; then
- echo "No need to build the docker container"
- exit 0
- else
- echo "Building"
- fi
rules:
- if: $CI_COMMIT_TAG
- if: $CI_PIPELINE_SOURCE == "web"
upload-to-docker-hub:
image: ubuntu:22.04
stage: publish
variables:
KUBERNETES_CPU_LIMIT: 4
KUBERNETES_CPU_REQUEST: 2
KUBERNETES_MEMORY_LIMIT: 8Gi
KUBERNETES_MEMORY_REQUEST: 2Gi
needs: [configure-docker-image, build-docker-image]
before_script:
- if [ "$BUILD_DOCKER_CONTAINER" == "false" ]; then
- echo "No need to build the docker container"
- exit 0
- else
- echo "Building"
- fi
script:
- apt-get update && apt-get install -y jq curl podman
- export HUB_REPO_NAME="neurodamus"
- |
TOKEN=$(curl -s --header "Content-Type:application/json" --request POST --data "{\"username\":\"${DOCKER_HUB_USER}\", \"password\":\"${DOCKER_HUB_AUTH_TOKEN}\"}" https://hub.docker.com/v2/users/login/ | jq -r .token)
- echo "Pushing to docker hub repo docker://docker.io/bluebrain/${HUB_REPO_NAME}"
- podman login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY_IMAGE}
- podman login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_AUTH_TOKEN} docker.io
- podman pull ${CI_REGISTRY_IMAGE}:${REGISTRY_IMAGE_TAG}
- echo podman push ${CI_REGISTRY_IMAGE}:${REGISTRY_IMAGE_TAG} docker://docker.io/bluebrain/${HUB_REPO_NAME}:${REGISTRY_IMAGE_TAG}
- curl -s --header "Content-Type:application/json" --header "Authorization:JWT ${TOKEN}" -X DELETE "https://hub.docker.com/v2/repositories/bluebrain/neurodamus/tags/latest/"
- podman --log-level=debug push ${CI_REGISTRY_IMAGE}:${REGISTRY_IMAGE_TAG} docker://docker.io/bluebrain/${HUB_REPO_NAME}:${REGISTRY_IMAGE_TAG}
- podman --log-level=debug push ${CI_REGISTRY_IMAGE}:${REGISTRY_IMAGE_TAG} docker://docker.io/bluebrain/${HUB_REPO_NAME}:latest
rules:
- if: $CI_COMMIT_TAG
- if: $CI_PIPELINE_SOURCE == "web"
when: manual