Release 1.3.0

CITATION.cff

@@ -1,8 +1,8 @@
 cff-version: "1.2.0"
 message: "If you use this software, please cite it using these metadata."
 title: ElasticBLAST
-version: "1.2.0"
-date-released: 2023-11-27
+version: "1.3.0"
+date-released: 2024-07-17
 license: "NCBI Public Domain"
 repository-code: "https://github.com/ncbi/elastic-blast/"
 url: "https://blast.ncbi.nlm.nih.gov/doc/elastic-blast/"

docker-blast/Dockerfile

@@ -18,7 +18,7 @@
 #   
 # Please cite NCBI in any work or product based on this material.
 
-FROM ncbi/blast:2.15.0 as blast
+FROM ncbi/blast:2.16.0 as blast
 ARG version
 LABEL Description="NCBI BLAST" Vendor="NCBI/NLM/NIH" Version=${version} [email protected]
 
@@ -29,6 +29,7 @@ COPY requirements.txt .
 
 RUN apt-get -y -m update && \
     apt-get install -y python3 python3-pip time parallel vmtouch curl wget unzip && \
+    pip3 --version && python3 -m pip --version && \
 	rm -rf /var/lib/apt/lists/*  
 
 RUN python3 -m pip install --no-cache-dir --upgrade pip && \

docker-blast/Dockerfile-build-from-local-sources

@@ -18,7 +18,7 @@
 #   
 # Please cite NCBI in any work or product based on this material.
 
-FROM ncbi/blast:latest as blast
+FROM ncbi/blast:2.16.0 as blast
 ARG version
 LABEL Description="NCBI BLAST" Vendor="NCBI/NLM/NIH" Version=${version} [email protected]
 

docker-blast/Makefile

@@ -30,7 +30,7 @@ GCP_IMG?=gcr.io/ncbi-sandbox-blast/${IMG}
 AWS_SERVER?=public.ecr.aws/i6v3i0i9
 AWS_IMG?=${AWS_SERVER}/elasticblast-elb
 AWS_REGION?=us-east-1
-VERSION?=1.3.1
+VERSION?=1.3.2
 
 ifeq (, $(shell which vmtouch 2>/dev/null))
 NOVMTOUCH?=--no-vmtouch

docker-blast/README.md

@@ -8,3 +8,4 @@ various repositories.
 
 If you have `docker` available, run `make build` to build the image, and `make
 check` to test it locally.
+

docker-blast/test-docker-image-aws.yaml

@@ -12,7 +12,9 @@ steps:
 - name: '${_IMG}'
   args: ['aws', 's3', '--no-sign-request', 'ls', 's3://ncbi-blast-databases/latest-dir']
 - name: '${_IMG}'
-  args: ['gcloud', '--version']
+  args: ['gsutil', '--version']
+- name: '${_IMG}'
+  args: ['gsutil', 'ls', 'gs://blast-db']
 - name: '${_IMG}'
   args: ['printenv', 'BLASTDB', 'PATH']
 - name: '${_IMG}'

docker-blast/test-docker-image-gcp.yaml

@@ -12,7 +12,9 @@ steps:
 - name: 'gcr.io/${PROJECT_ID}/${_IMG}:${_TAG}'
   args: ['aws', 's3', '--no-sign-request', 'ls', 's3://ncbi-blast-databases/latest-dir']
 - name: 'gcr.io/${PROJECT_ID}/${_IMG}:${_TAG}'
-  args: ['gcloud', '--version']
+  args: ['gsutil', '--version']
+- name: 'gcr.io/${PROJECT_ID}/${_IMG}:${_TAG}'
+  args: ['gsutil', 'ls', 'gs://blast-db']
 - name: 'gcr.io/${PROJECT_ID}/${_IMG}:${_TAG}'
   args: ['printenv', 'BLASTDB', 'PATH']
 - name: 'gcr.io/${PROJECT_ID}/${_IMG}:${_TAG}'

docker-janitor/Dockerfile-build-from-local-sources.gcp

@@ -32,13 +32,12 @@ COPY elastic-blast-janitor.sh /usr/bin/
 RUN chmod +x /usr/bin/elastic-blast-janitor.sh && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel curl && \
-    pip3 install --no-cache-dir --upgrade pip && \
-	curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl && \
-	chmod +x ./kubectl && \
-	mv kubectl /usr/bin/ && \
+    pip3 install --no-cache-dir --upgrade --break-system-packages pip && \
     mkdir /var/elastic-blast && \
     rm -rf /var/cache/apk/*
 
+RUN gcloud -q components install kubectl
+
 COPY src/ /var/elastic-blast/src/
 COPY bin/ /var/elastic-blast/bin/
 COPY requirements/ /var/elastic-blast/requirements/
@@ -47,6 +46,6 @@ COPY setup.cfg_cloud /var/elastic-blast/setup.cfg
 
 WORKDIR /var/elastic-blast
 
-RUN pip3 install . -r requirements/test.txt
+RUN pip3 install . -r requirements/test.txt --break-system-packages
 
 CMD ["/usr/bin/elastic-blast-janitor.sh"]

docker-janitor/Dockerfile.gcp

@@ -33,8 +33,8 @@ COPY elastic-blast-janitor.sh /usr/bin/
 RUN chmod +x /usr/bin/elastic-blast-janitor.sh && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel curl && \
-    pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir -r requirements.txt && rm -rf /var/cache/apk/* requirements.txt
+    pip3 install --no-cache-dir --upgrade --break-system-packages pip && \
+    pip3 install --no-cache-dir -r requirements.txt --break-system-packages && rm -rf /var/cache/apk/* requirements.txt
 
 RUN gcloud -q components install kubectl
 

docker-janitor/Makefile

@@ -27,7 +27,8 @@ SHELL=/bin/bash
 .PHONY: all pre-check check clean build publish gcp-build gcp-check gcp-clean
 
 IMG?=ncbi/elasticblast-janitor
-VERSION?=0.3.1
+VERSION?=0.3.2
+ELB_VERSION?=$(shell git describe --tags --abbrev=0)
 GCP_PROJECT?=$(shell gcloud config get-value project 2>/dev/null)
 GCP_TEST_BUCKET?=gs://elasticblast-test/query-split-run-test
 
@@ -64,7 +65,7 @@ gcp-build:
 gcp-build-from-local-sources:
 	rsync -a ../setup.py ../setup.cfg_cloud ../src ../bin ../requirements ${PWD}/
 	sed -i~ -e '/^value = $${VERSION}/d;' setup.cfg_cloud
-	echo "value = ${VERSION}" >> setup.cfg_cloud
+	echo "value = ${ELB_VERSION}" >> setup.cfg_cloud
 	-gcloud builds submit --config cloudbuild.yaml --substitutions _VERSION=${VERSION},_IMG=${IMG},_DOCKERFILE='Dockerfile-build-from-local-sources.gcp'
 	rm -fr src bin requirements setup.cfg_cloud setup.py
 

docker-job-submit/Dockerfile-build-from-local-sources.aws

@@ -31,7 +31,7 @@ COPY submit_jobs.py /usr/bin/
 RUN chmod +x /usr/bin/submit_jobs.py && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel && \
-    pip3 install --no-cache-dir --upgrade pip && \
+    pip3 install --no-cache-dir --upgrade --break-system-packages pip && \
     mkdir -p /var/elastic-blast && \
     rm -rf /var/cache/apk/*
 
@@ -43,6 +43,6 @@ COPY setup.cfg_cloud /var/elastic-blast/setup.cfg
 
 WORKDIR /var/elastic-blast
 
-RUN pip3 install . -r requirements/base.txt
+RUN pip3 install . -r requirements/base.txt --break-system-packages
 
 CMD ["/usr/bin/submit_jobs.py", "--help"]

docker-job-submit/Dockerfile.aws

@@ -27,8 +27,8 @@ COPY submit_jobs.py /usr/bin/
 RUN chmod +x /usr/bin/submit_jobs.py && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel curl unzip && \
-    pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir -r requirements.txt && rm -rf /var/cache/apk/* requirements.txt
+    pip3 install --no-cache-dir --upgrade --break-system-packages  pip && \
+    pip3 install --no-cache-dir -r requirements.txt --break-system-packages && rm -rf /var/cache/apk/* requirements.txt
 
 LABEL Description="NCBI ElasticBLAST Cloud Job Submission Module"
 LABEL Version=${version}

docker-job-submit/Makefile

@@ -28,7 +28,8 @@ SHELL=/bin/bash
 .PHONY: all pre-check check clean build publish gcp-build gcp-check gcp-clean
 
 IMG?=ncbi/elasticblast-job-submit
-VERSION?=4.0.2
+VERSION?=4.0.3
+ELB_VERSION?=$(shell git describe --tags --abbrev=0)
 GCP_PROJECT?=$(shell gcloud config get-value project 2>/dev/null)
 GCP_TEST_BUCKET?=gs://elasticblast-test/cloud-job-submission
 AWS_REGION?=us-east-1
@@ -73,7 +74,7 @@ aws-build:
 aws-build-from-local-sources:
 	rsync -a ../setup.py ../setup.cfg_cloud ../src ../bin ../requirements ${PWD}/
 	sed -i~ -e '/^value = $${VERSION}/d;' setup.cfg_cloud
-	echo "value = ${VERSION}" >> setup.cfg_cloud
+	echo "value = ${ELB_VERSION}" >> setup.cfg_cloud
 	-gcloud builds submit --config awscloudbuild.yaml --substitutions _SERVER=${AWS_SERVER},TAG_NAME=${VERSION},_IMG=${AWS_IMG},_DOCKERFILE=Dockerfile-build-from-local-sources.aws,_AWS_ECR_PASSWD="`aws ecr-public get-login-password --region ${AWS_REGION}`"
 	rm -fr src bin requirements setup.cfg_cloud setup.py
 

docker-job-submit/cloud-job-submit.sh

@@ -201,7 +201,10 @@ done
 
 # label the new persistent disk
 export pv=$(${KUBECTL} get -f pvc-rom.yaml -o jsonpath='{.spec.volumeName}')
-jq -n --arg dd $pv '[$dd]' | gsutil cp - ${ELB_RESULTS}/${ELB_METADATA_DIR}/$ELB_DISK_ID_FILE
+export vs=snapshot-$(${KUBECTL} get -f /templates/volume-snapshot.yaml -o jsonpath='{.metadata.uid}')
+echo "PV: $pv"
+echo "Volume snapshot: $vs"
+jq -n --arg dd $pv --arg ss $vs '{"disks": [$dd], "snapshots": [$ss]}' | gsutil -qm cp - ${ELB_RESULTS}/${ELB_METADATA_DIR}/$ELB_DISK_ID_FILE
 gcloud compute disks update $pv --update-labels ${ELB_LABELS} --zone ${ELB_GCP_ZONE} --project ${ELB_GCP_PROJECT}
 
 # delete snapshot
@@ -214,6 +217,6 @@ if gcloud compute disks describe $pv_rwo --zone $ELB_GCP_ZONE ; then
     sleep 10
 
     if gcloud compute disks describe $pv_rwo --zone $ELB_GCP_ZONE ; then
-        jq -n --arg d1 $pv_rwo --arg d2 $pv '[d1, d2]' | gsutil cp - ${ELB_RESULTS}/${ELB_METADATA_DIR}/$ELB_DISK_ID_FILE
+        jq -n --arg d1 $pv_rwo --arg d2 $pv --arg ss $vs '{"disks": [$d1, $d2], "snapshots": [$ss]}' | gsutil -qm cp - ${ELB_RESULTS}/${ELB_METADATA_DIR}/$ELB_DISK_ID_FILE
     fi
 fi

docker-qs/Dockerfile

@@ -28,8 +28,8 @@ COPY run.sh /usr/bin/
 RUN chmod +x /usr/bin/run.sh && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel curl unzip && \
-    pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir -r requirements.txt && rm -rf /var/cache/apk/* requirements.txt
+    pip3 install --no-cache-dir --upgrade pip --break-system-packages && \
+    pip3 install --no-cache-dir -r requirements.txt --break-system-packages && rm -rf /var/cache/apk/* requirements.txt
 
 LABEL Description="NCBI ElasticBLAST Query Splitting Module"
 LABEL Version=${version}

docker-qs/Dockerfile-build-from-local-sources

@@ -34,8 +34,8 @@ COPY run.sh /usr/bin/
 RUN chmod +x /usr/bin/run.sh && \
     apk -U upgrade && \
     apk add --no-cache bash python3 py3-pip py3-wheel && \
-    pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir -r requirements.txt && \
+    pip3 install --no-cache-dir --upgrade pip --break-system-packages && \
+    pip3 install --no-cache-dir -r requirements.txt --break-system-packages && \
     mkdir -p /var/elastic-blast && \
     rm -rf /var/cache/apk/* requirements.txt
 

docker-qs/Makefile

@@ -29,6 +29,7 @@ SHELL=/bin/bash
 
 IMG?=ncbi/elasticblast-query-split
 VERSION?=0.1.4
+ELB_VERSION=?=$(shell git describe --tags --abbrev=0)
 GCP_PROJECT?=$(shell gcloud config get-value project 2>/dev/null)
 GCP_TEST_BUCKET?=gs://elasticblast-test/query-split-run-test
 AWS_REGION?=us-east-1
@@ -84,7 +85,7 @@ aws-build:
 aws-build-from-local-sources:
 	rsync -a ../setup.py ../setup.cfg_cloud ../src ../bin ../requirements ${PWD}/
 	sed -i~ -e '/^value = $${VERSION}/d;' setup.cfg_cloud
-	echo "value = ${VERSION}" >> setup.cfg_cloud
+	echo "value = ${ELB_VERSION}" >> setup.cfg_cloud
 	gcloud builds submit --config awscloudbuild.yaml --substitutions _SERVER=${AWS_SERVER},TAG_NAME=${VERSION},_IMG=${AWS_IMG},_DOCKERFILE=Dockerfile-build-from-local-sources,_AWS_ECR_PASSWD="`aws ecr-public get-login-password --region ${AWS_REGION}`"
 	rm -fr src bin requirements setup.cfg_cloud setup.py
 

docker-qs/README.md

@@ -10,3 +10,5 @@ various repositories.
 
 If you have `docker` available, run `make build` to build the image, and `make
 check` to test it locally.
+
+You will need credentials for NCBI-AWS-ELASTICBLAST-OPERATIONS to push to AWS ECR, see *To get NCBI AWS credentials in NCBI-AWS-ELASTICBLAST-OPERATIONS* in  [README-ncbi.md](../README-ncbi.md).

requirements/base.txt

@@ -1,11 +1,10 @@
-wheel==0.38.4
+wheel==0.43.0
 setuptools==70.0.0
-importlib-resources==5.10.2
-importlib-metadata==6.0.0
-pex==2.1.152
-boto3==1.29.1
-botocore==1.32.1
+importlib-resources==6.1.1
+importlib-metadata==7.0.0
+pex==2.9.0
+boto3==1.34.141
+botocore==1.34.141
 awslimitchecker==12.0.0
-tenacity==8.2.3
-dataclasses-json==0.6.2
-types-pkg-resources==0.1.3
+tenacity==8.5.0
+dataclasses-json==0.6.7

requirements/test.txt

@@ -1,14 +1,14 @@
 -r base.txt
 
-pytest==7.4.3
-pytest-cov==4.1.0
-pytest-mock==3.12.0
+pytest==8.2.2
+pytest-cov==5.0.0
+pytest-mock==3.14.0
 teamcity-messages==1.32
-mypy==1.7.0
+mypy==1.10.1
 pylint==2.7.4
 tox==4.4.12
-virtualenv==20.21.0
-yamllint==1.33.0
-moto==4.2.8
-docker==6.1.3
-cfn-lint==0.83.3
+virtualenv==20.24.5
+yamllint==1.35.1
+moto==4.2.14
+docker==7.1.0
+cfn-lint==1.5.1

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = elastic_blast
-description = ElasticBLAST is a cloud-based tool to perform your BLAST searches faster and make you more effective
+description = ElasticBLAST speeds up your work by distributing your BLAST+ searches across multiple cloud instances. The ability to scale resources in this way allows larger numbers of queries to be searched in a shorter time than you could with BLAST+ on a single host.  Use of the cloud also allows you to collaborate with colleagues, allowing the sharing of results, datasets and pipelines on a common platform. The National Center for Biotechnology Information ([NCBI](https://www.ncbi.nlm.nih.gov)), part of the National Library of Medicine at the NIH, developed and maintains ElasticBLAST.
 long_description = file:README.md
 long_description_content_type = text/markdown
 maintainer = NCBI

src/elastic_blast/aws_traits.py

@@ -31,7 +31,7 @@
 from .util import UserReportError, check_aws_region_for_invalid_characters
 from .base import InstanceProperties, PositiveInteger, MemoryStr
 from .constants import ELB_DFLT_AWS_REGION, INPUT_ERROR, PERMISSIONS_ERROR
-from .constants import DEPENDENCY_ERROR
+from .constants import DEPENDENCY_ERROR, ELB_DFLT_AWS_REGION
 
 
 def create_aws_config(region: Optional[str] = None) -> Config:
@@ -114,7 +114,8 @@ def get_instance_type_offerings(region: str) -> List[str]:
 
 def get_suitable_instance_types(min_memory: MemoryStr,
                                 min_cpus: PositiveInteger,
-                                instance_types: Optional[List[str]] = None) -> List[Any]:
+                                instance_types: Optional[List[str]] = None,
+                                region: str = ELB_DFLT_AWS_REGION) -> List[Any]:
     """Get a list of instance type descriptions with at least min_memory and
     number of CPUs
 
@@ -126,7 +127,7 @@ def get_suitable_instance_types(min_memory: MemoryStr,
     Returns:
         A list of instance type descriptions for instance types that satisfy
         the above constraints"""
-    ec2 = boto3.client('ec2')
+    ec2 = boto3.client('ec2', region_name=region)
 
     # select only 64-bit CPUs
     filters = [{'Name': 'processor-info.supported-architecture',

src/elastic_blast/commands/submit.py

@@ -41,8 +41,9 @@
 from elastic_blast.split import FASTAReader
 from elastic_blast.gcp import check_cluster as gcp_check_cluster
 from elastic_blast.gcp_traits import get_machine_properties
-from elastic_blast.util import get_blastdb_size, UserReportError
+from elastic_blast.util import check_user_provided_blastdb_exists, UserReportError
 from elastic_blast.util import get_resubmission_error_msg
+from elastic_blast.util import ElbSupportedPrograms
 from elastic_blast.constants import ELB_AWS_JOB_IDS, ELB_METADATA_DIR, ELB_STATE_DISK_ID_FILE, QuerySplitMode
 from elastic_blast.constants import ELB_QUERY_BATCH_DIR, BLASTDB_ERROR, INPUT_ERROR
 from elastic_blast.constants import PERMISSIONS_ERROR, CLUSTER_ERROR, CSP, QUERY_LIST_EXT
@@ -118,7 +119,7 @@ def submit(args, cfg, clean_up_stack):
     cfg.validate(ElbCommand.SUBMIT, dry_run)
 
     # For now, checking resources is only implemented for AWS
-    if cfg.cloud_provider.cloud == CSP.AWS:
+    if cfg.cloud_provider.cloud == CSP.AWS and os.getenv('TEAMCITY_VERSION') is None:
         check_resource_quotas(cfg)
 
     if check_running_cluster(cfg):
@@ -151,7 +152,7 @@ def submit(args, cfg, clean_up_stack):
     # check database availability
     gcp_prj = None if cfg.cloud_provider.cloud == CSP.AWS else cfg.gcp.get_project_for_gcs_downloads()
     try:
-        get_blastdb_size(cfg.blast.db, cfg.cluster.db_source, gcp_prj)
+        check_user_provided_blastdb_exists(cfg.blast.db, ElbSupportedPrograms().get_db_mol_type(cfg.blast.program), cfg.cluster.db_source, gcp_prj)
     except ValueError as err:
         raise UserReportError(returncode=BLASTDB_ERROR, message=str(err))
 

src/elastic_blast/constants.py

@@ -210,10 +210,10 @@ def __str__(self):
 ELB_DFLT_AWS_REGION = 'us-east-1'
 ELB_UNKNOWN_GCP_PROJECT = 'elb-unknown-gcp-project'
 
-ELB_DOCKER_VERSION = '1.3.1'    # ElasticBLAST 1.2.0 uses BLAST+ 2.15.0
+ELB_DOCKER_VERSION = '1.3.2'    # ElasticBLAST 1.3.0 uses BLAST+ 2.16.0
 ELB_QS_DOCKER_VERSION = '0.1.4'
-ELB_JANITOR_DOCKER_VERSION = '0.3.1'
-ELB_JOB_SUBMIT_DOCKER_VERSION = '4.0.2'
+ELB_JANITOR_DOCKER_VERSION = '0.3.2'
+ELB_JOB_SUBMIT_DOCKER_VERSION = '4.0.3'
 
 ELB_DOCKER_IMAGE_GCP = f'gcr.io/ncbi-sandbox-blast/ncbi/elb:{ELB_DOCKER_VERSION}'
 ELB_DOCKER_IMAGE_AWS = f'public.ecr.aws/ncbi-elasticblast/elasticblast-elb:{ELB_DOCKER_VERSION}'