[WIP] Update CI.yaml

.github/workflows/CI.yaml

@@ -86,23 +86,6 @@ jobs:
         /sbin/vboxconfig;
         vagrant plugin install --local;
         vagrant plugin install --local;
-        gpg --quick-generate-key --batch --passphrase '' "Lyra Silvertongue (zcash gitian) <[email protected]>"
-        echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env;
-        echo "GPG_KEY_NAME=lyra.silvertongue" >> .env;
-        git config --global user.name "Lyra Silvertongue"
-        git config --global user.email "[email protected]"
-        direnv allow;
-        direnv exec \$(pwd) vagrant up zcash-build;
-        vagrant ssh zcash-build -c "gpg --quick-generate-key --batch --passphrase '' \"Lyra Silvertongue (zcash gitian) <[email protected]>\" || echo ''"
-        vagrant ssh zcash-build -c ./gitian-parallel-build.sh || exit 1
-        vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/lyra.silvertongue/*.assert" > assert.txt
-        tr -d \$'\r' < assert.txt > assert2.txt
-        echo "#### sigs ####"
-        for i in \$(cat assert2.txt | grep -E "zcash-*"  | grep -v git: | sed 's/    //g' | sed 's/  /-->/g'); do
-          echo \$i
-        done
-        export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r')
-        for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done
 
         # get keys
         gsutil -q rm -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1 || echo ""
@@ -118,6 +101,28 @@ jobs:
             --ciphertext-file encrypted_gpg.kms
         cd \$current_dir
         gpg --import \$HOME/private.pgp
+        export key=\$(gpg --list-secret-keys --keyid-format=long [email protected] | head -n 2 | grep -v sec)
+        echo "GPG_KEY_ID=\$(gpg --list-keys --with-fingerprint --with-colons | grep fpr: | head -n 1 | sed 's/fpr://g' | sed 's/://g')" >> .env;
+        echo "GPG_KEY_NAME=sysadmin" >> .env;
+
+        git config --global user.name "sysadmin"
+        git config --global user.email "[email protected]"
+
+        # build
+        direnv allow;
+        direnv exec \$(pwd) vagrant up zcash-build;
+        vagrant scp \$HOME/private.pgp :
+        vagrant ssh zcash-build -c "gpg --import private.pgp"
+        vagrant ssh zcash-build -c ./gitian-parallel-build.sh || echo ""
+        vagrant ssh zcash-build -c "head -n 8 gitian.sigs/\$VERSION*/sysadmin/*.assert" > assert.txt
+        tr -d \$'\r' < assert.txt > assert2.txt
+        echo "#### sigs ####"
+        for i in \$(cat assert2.txt | grep -E "zcash-*"  | grep -v git: | sed 's/    //g' | sed 's/  /-->/g'); do
+          echo \$i
+        done
+        export OS=\$(vagrant ssh zcash-build -c "ls zcash-binaries/\$VERSION" | tr -d '\r')
+        for i in \$OS; do vagrant ssh zcash-build -c "mkdir \$i; tar Cxvzf \$i zcash-binaries/*/\$i/zcash-*-linux64.tar.gz"; done
+
         vagrant scp :gitian.sigs .
         for i in \$OS;
         do
@@ -142,6 +147,8 @@ jobs:
           echo #### zcashd --version #### 
           docker exec -it \$i bash -c "zcashd --version"
         done
+
+        # sign binaries
         vagrant scp :/home/vagrant/zcash-binaries ./
         for i in \$OS;
         do
@@ -154,24 +161,25 @@ jobs:
           done
           gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *debug-debian-\$i.tar.gz
           gpg -u [email protected] --armor --digest-algo SHA256 --detach-sign *linux64-debian-\$i.tar.gz
+          rm -rf zcash-$(echo $VERSION | sed 's/v//g').tar.gz
+          gsutil -q -m rsync -r ./ gs://download-downloads/
           cd \$current_dir
         done
         export final_version=\$(cat assert2.txt | awk '{print \$2}' | grep "desc.yml" | head -n 1 | sed 's/-desc.yml//g')
         gsutil -q -m rsync -r ./debs gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/debs
         gsutil -q -m rsync -r ./zcash-binaries gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/zcash-binaries
+
         apt install aptly -y >/dev/null
 
         # generate apt
         mkdir aptserver
         cd aptserver
         gsutil -q -m cp -r gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/pool/main/z/zcash/ .
         cd zcash
-        cp -a ../../debs/buster/zcbuild/*.deb \$final_version-amd64-buster.deb
         cp -a ../../debs/bullseye/zcbuild/*.deb \$final_version-amd64-bullseye.deb
-        cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb || echo ""
-        ls \$final_version-amd64-buster.deb || exit 1
+        cp -a ../../debs/bookworm/zcbuild/*.deb \$final_version-amd64-bookworm.deb
         ls \$final_version-amd64-bullseye.deb || exit 1
-        ls \$final_version-amd64-bookworm.deb || echo ""
+        ls \$final_version-amd64-bookworm.deb || exit 1
 
         aptly repo create --distribution buster --comment "" --component main zcash_buster_amd64_repo
         aptly repo create --distribution bullseye --comment "" --component main zcash_bullseye_amd64_repo
@@ -194,7 +202,6 @@ jobs:
         aptly snapshot create bullseye_snapshot from repo zcash_bullseye_amd64_repo
         aptly snapshot create stretch_snapshot from repo zcash_stretch_amd64_repo
 
-        export key=\$(gpg --list-secret-keys --keyid-format=long [email protected] | head -n 2 | grep -v sec)
         aptly publish snapshot --distribution buster --component main --architectures amd64 --gpg-key="\$key" --passphrase="" buster_snapshot
         aptly publish snapshot --distribution bookworm --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bookworm_snapshot
         aptly publish snapshot --distribution bullseye --component main --architectures amd64 --gpg-key="\$key" --passphrase="" bullseye_snapshot
@@ -215,46 +222,50 @@ jobs:
         cp -a /root/.aptly/public /var/www/
         chown -R www-data:www-data /var/www
         /etc/init.d/nginx restart
+        echo "debug 1"
         mkdir \$HOME/mirror
         cd \$HOME/mirror
         wget -q -r 127.0.0.1
-
         cp \$HOME/public.asc \$HOME/mirror/127.0.0.1/zcash.asc
         cd \$HOME/mirror
         gsutil -q -m rsync -r ./127.0.0.1 gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-packages/127.0.0.1
         cd 127.0.0.1
-        if ! [[ ${array[2]} == *"-rc"* ]]; then
+        if [[ ! ${array[2]} =~ -rc ]] && [[ ${array[2]} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
           gsutil -q -m rsync -r ./ gs://${{ secrets.GCP_PROJECT_ID_PROD }}-apt-server/
         fi
         echo "script finished"
         EOF
 
         export FAIL=0
         chmod +x ./script.sh
+        gcloud compute scp ./script.sh --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || (echo "error 1" && export FAIL=1)
+        gcloud compute scp --recurse $(pwd) --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || (echo "error 2" && export FAIL=1)
+        gcloud compute ssh --zone "us-central1-a" "test-gitian-$random"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || (echo "error 3" && export FAIL=1)
+
+        if [ $FAIL -eq 1 ]; then 
+          echo "error"
+          #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all
+          exit 1; 
+        fi 
 
-        gcloud compute scp ./script.sh --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random: || export FAIL=1
-        gcloud compute scp --recurse $(pwd) --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:~/source || export FAIL=1
-
-        gcloud compute ssh --zone "us-central1-a" "test-gitian-$random"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --command="bash -i -c 'sudo -s ./script.sh'" -- -t || export FAIL=1
+        # curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' # debug
 
-        gcloud compute scp --recurse --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs . || export FAIL=1
+        gcloud compute scp --recurse --zone "us-central1-a"  --tunnel-through-iap --project "${{ secrets.GCP_PROJECT_ID_PROD }}" test-gitian-$random:/home/sa_*/source/gitian.sigs .
 
-        curl -s --request POST --url https://api.bunny.net/pullzone/${{ secrets.BUNNY_RESOURCE }}/purgeCache --header 'content-type: application/json' --header 'AccessKey: ${{ secrets.BUNNY_API_KEY }}' || export FAIL=1
-
-        rm -rf gitian.sigs/.git || export FAIL=1
-        if ! [[ ${array[2]} == *"-rc"* ]]; then
+        rm -rf gitian.sigs/.git 
+          if [[ ! ${array[2]} =~ -rc ]] && [[ ${array[2]} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
           mkdir $HOME/.ssh || echo ""
-          ssh-keyscan github.com >> $HOME/.ssh/known_hosts || export FAIL=1
+          ssh-keyscan github.com >> $HOME/.ssh/known_hosts 
           echo "${{ secrets.BOT_SSH_KEY }}" > $HOME/.ssh/id_rsa 
           chmod 600 $HOME/.ssh/id_rsa
-          git clone [email protected]:zcash/gitian.sigs.git sigs || export FAIL=1
+          git clone [email protected]:zcash/gitian.sigs.git sigs 
           cp -a gitian.sigs/* sigs/
           cd sigs
           git config --global user.name "ECC-CI"
           git config --global user.email "${{ secrets.BOT_EMAIL }}"
           git add .
-          git commit -am "${{ github.event.label.name }}" || export FAIL=1
-          git push || export FAIL=1
+          git commit -am "${{ github.event.label.name }}" 
+          git push # debug
         fi
-        gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all
-        if [ $FAIL -eq 1 ]; then exit 1; fi 
+        echo "last step"
+        #gcloud compute instances delete "test-gitian-$random" --project "${{ secrets.GCP_PROJECT_ID_PROD }}" --zone "us-central1-a" --delete-disks=all # debug

roles/gitian/templates/gitian-parallel-build.sh

@@ -215,11 +215,9 @@ then
             echo ""
             echo "Compiling variant: ${VERSION}_${suite}"
             echo ""
-            #workaround python and python3 in buster
-            if [[ $suite = "buster" ]]
-            then
-                sed -i -e 's/- "python3"/- "python"/g' -e 's/- "python-is-python3"//g' ${suite_dir_path}/gitian-linux-parallel.yml;
-            fi
+            #workaround python and python3
+            sed -i '/- "python3"/c\- "python3"\n- "python-is-python3"' ${suite_dir_path}/gitian-linux-parallel.yml;
+
             ./bin/gbuild --fetch-tags -j ${proc} -m ${mem} --commit zcash=${COMMIT} --url zcash=${url} ${suite_dir_path}/gitian-linux-parallel.yml
             ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}_${suite} --destination ${gitian_sigs_repo_path}/ ${suite_dir_path}/gitian-linux-parallel.yml
 
@@ -266,4 +264,4 @@ then
     done
 
     popd
-fi
+fi