Reload data on a demo server #11
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reload data on a demo server | |
| on: | |
| workflow_call: | |
| inputs: | |
| environment: | |
| description: Environment in which to reload data | |
| type: string | |
| required: true | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: Environment in which to reload data | |
| type: environment | |
| required: true | |
| jobs: | |
| reload: | |
| name: Clean up ${{ inputs.environment }} demo server | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Get instance address | |
| id: ec2-describe-instances | |
| run: | | |
| INSTANCE_ADDRESS=$(aws ec2 describe-instances \ | |
| --instance-ids ${{ vars.INSTANCE_ID }} \ | |
| --query "Reservations[*].Instances[*].[PublicDnsName]" \ | |
| --output text) | |
| echo "INSTANCE_ADDRESS=$INSTANCE_ADDRESS" >> "$GITHUB_OUTPUT" | |
| - name: Set up SSH | |
| run: | | |
| mkdir --parents ~/.ssh | |
| echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/staging | |
| chmod 600 ~/.ssh/staging | |
| cat >>~/.ssh/config <<END | |
| Host staging | |
| HostName ${{ steps.ec2-describe-instances.outputs.INSTANCE_ADDRESS }} | |
| User ec2-user | |
| IdentityFile ~/.ssh/staging | |
| END | |
| ssh-keyscan -H ${{ steps.ec2-describe-instances.outputs.INSTANCE_ADDRESS }} >> ~/.ssh/known_hosts | |
| - name: Setup and Reload | |
| run: | | |
| xid=`date +"%Y-%m-%dT%H-%M-%S"` | |
| cat <<'EOF' > cleanup_apollo.$xid.sh | |
| set -e | |
| set -u | |
| set -o pipefail | |
| set -x | |
| # Download data | |
| demoDataDir=${{ '${HOME}' }}/reload_demo_data/demoData | |
| rm -rf $demoDataDir | |
| mkdir -p $demoDataDir | |
| cd $demoDataDir | |
| wormbase='https://ftp.ebi.ac.uk/pub/databases/wormbase/parasite/releases/WBPS19/species' | |
| curl -O -s -L $wormbase/schistosoma_haematobium/TD2_PRJEB44434/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.genomic.fa.gz | |
| curl -O -s -L $wormbase/schistosoma_mansoni/PRJEA36577/schistosoma_mansoni.PRJEA36577.WBPS19.genomic.fa.gz | |
| curl -O -s -L $wormbase/schistosoma_haematobium/TD2_PRJEB44434/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.annotations.gff3.gz | |
| curl -O -s -L $wormbase/schistosoma_mansoni/PRJEA36577/schistosoma_mansoni.PRJEA36577.WBPS19.annotations.gff3.gz | |
| gunzip *.gz | |
| # Setup docker | |
| if [ "${{ inputs.environment }}" = "staging" ]; then | |
| env=devel | |
| elif [ "${{ inputs.environment }}" = "prod" ]; then | |
| env=latest | |
| else | |
| echo "Invalid environment: ${{ inputs.environment }}" >&2 | |
| fi | |
| docker pull ghcr.io/gmod/apollo-cli:${env} | |
| cfgDir=${{ '${HOME}' }}/reload_demo_data/config | |
| mkdir -p $cfgDir | |
| touch $cfgDir/config.yml | |
| function apollo () { | |
| docker run --network host -v $cfgDir:/root/.config/apollo-cli -v $demoDataDir:/data ghcr.io/gmod/apollo-cli:${env} "$@" | |
| } | |
| cfg="--config-file /root/.config/apollo-cli/config.yml --profile auto" | |
| # Configure Apollo | |
| apollo config $cfg address http://localhost/apollo | |
| apollo config $cfg accessType root | |
| apollo config $cfg rootCredentials.username ${{ secrets.ROOT_USER_NAME }} | |
| apollo config $cfg rootCredentials.password ${{ secrets.ROOT_USER_PASSWORD }} | |
| apollo login $cfg | |
| # Cleanup & import | |
| asm=`apollo assembly get $cfg | jq -r '.[] | ._id'` | |
| apollo assembly delete $cfg -v -a $asm | |
| apollo assembly add-fasta $cfg -i /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.genomic.fa -a schistosoma_haematobium -f | |
| apollo assembly add-fasta $cfg -i /data/schistosoma_mansoni.PRJEA36577.WBPS19.genomic.fa -a schistosoma_mansoni -f | |
| apollo feature import $cfg -i /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.annotations.gff3 -a schistosoma_haematobium -d | |
| apollo feature import $cfg -i /data/schistosoma_mansoni.PRJEA36577.WBPS19.annotations.gff3 -a schistosoma_mansoni -d | |
| echo `date +"%Y-%m-%dT%H-%M-%S"` 'OK' | |
| EOF | |
| logDir=reload_demo_data/logs | |
| ssh "${{ inputs.environment }}" -f "mkdir -p ${logDir}" | |
| rsync -av cleanup_apollo.$xid.sh "${{ inputs.environment }}":${logDir}/ | |
| ssh "${{ inputs.environment }}" -f "bash ${logDir}/cleanup_apollo.$xid.sh > ${logDir}/cleanup_apollo.$xid.out 2> ${logDir}/cleanup_apollo.$xid.err" |