From 92ed985289f0ac6f5bd394b73eda343e8eda9349 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 30 Sep 2024 16:42:43 +0200 Subject: [PATCH 1/9] Updated the 03-annotation/lablogs in exomeeb, exometrio and wgstrio services to have all the necessary -bind routes --- .../exomeeb/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog | 2 +- .../exometrio/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog | 2 +- .../wgstrio/ANALYSIS/ANALYSIS01_GENOME/03-annotation/lablog | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bu_isciii/templates/exomeeb/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog b/bu_isciii/templates/exomeeb/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog index 61892d9b..ef8d93ad 100644 --- a/bu_isciii/templates/exomeeb/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog +++ b/bu_isciii/templates/exomeeb/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog @@ -22,7 +22,7 @@ echo "srun --partition short_idx --time 2:00:00 --chdir ${scratch_dir} --output ## 4-5. Lablog for annotating whole genome samples using Variant Effect Predictor (VEP). -echo "srun --partition short_idx --mem 100G --time 4:00:00 --chdir ${scratch_dir} --output logs/VEP.log --job-name VEP singularity exec -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/ensembl-vep:103.1--pl5262h4a94de4_2 vep --fasta /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/human_g1k_v37.fasta -i ${scratch_dir}/vep/variants_fil_mod.vcf -o ${scratch_dir}/vep/vep_annot.vcf --cache --offline --dir_cache /data/bi/references/eukaria/homo_sapiens/cache_vep/ --everything --dir_plugins /data/bi/references/eukaria/homo_sapiens/cache_vep/Plugins/ --assembly GRCh37 --tab --plugin dbNSFP,/data/bi/references/eukaria/homo_sapiens/cache_vep/custom_databases/dbNSFP/GRCh37/dbNSFP4.1a_grch37.gz,clinvar_id,clinvar_trait,clinvar_OMIM_id,clinvar_Orphanet_id,HGVSc_snpEff,HGVSp_snpEff,SIFT_score,SIFT_pred,Polyphen2_HDIV_score,Polyphen2_HDIV_pred,Polyphen2_HVAR_score,Polyphen2_HVAR_pred,MutationTaster_score,MutationTaster_pred,MutationAssessor_score,MutationAssessor_pred,FATHMM_score,FATHMM_pred,PROVEAN_score,PROVEAN_pred,VEST4_score,MetaSVM_score,MetaSVM_pred,MetaLR_score,MetaLR_pred,CADD_raw,CADD_phred,CADD_raw_hg19,CADD_phred_hg19,GERP++_NR,GERP++_RS,phyloP100way_vertebrate,phastCons100way_vertebrate &" > _02_vep_annotation.sh +echo "srun --partition short_idx --mem 100G --time 4:00:00 --chdir ${scratch_dir} --output logs/VEP.log --job-name VEP singularity exec -B /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/ -B /data/bi/references/eukaria/homo_sapiens/cache_vep/homo_sapiens -B /data/bi/references/eukaria/homo_sapiens/cache_vep/custom_databases/dbNSFP/GRCh37/ -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/ensembl-vep:103.1--pl5262h4a94de4_2 vep --fasta /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/human_g1k_v37.fasta -i ${scratch_dir}/vep/variants_fil_mod.vcf -o ${scratch_dir}/vep/vep_annot.vcf --cache --offline --dir_cache /data/bi/references/eukaria/homo_sapiens/cache_vep/ --everything --dir_plugins /data/bi/references/eukaria/homo_sapiens/cache_vep/Plugins/ --assembly GRCh37 --tab --plugin dbNSFP,/data/bi/references/eukaria/homo_sapiens/cache_vep/custom_databases/dbNSFP/GRCh37/dbNSFP4.1a_grch37.gz,clinvar_id,clinvar_trait,clinvar_OMIM_id,clinvar_Orphanet_id,HGVSc_snpEff,HGVSp_snpEff,SIFT_score,SIFT_pred,Polyphen2_HDIV_score,Polyphen2_HDIV_pred,Polyphen2_HVAR_score,Polyphen2_HVAR_pred,MutationTaster_score,MutationTaster_pred,MutationAssessor_score,MutationAssessor_pred,FATHMM_score,FATHMM_pred,PROVEAN_score,PROVEAN_pred,VEST4_score,MetaSVM_score,MetaSVM_pred,MetaLR_score,MetaLR_pred,CADD_raw,CADD_phred,CADD_raw_hg19,CADD_phred_hg19,GERP++_NR,GERP++_RS,phyloP100way_vertebrate,phastCons100way_vertebrate &" > _02_vep_annotation.sh echo "grep -v '^##' ./vep/vep_annot.vcf > ./vep/vep_annot_head.txt" > _03_merge_data1.sh echo "sed -i 's/#Uploaded_variation/ID/' ./vep/vep_annot_head.txt" >> _03_merge_data1.sh diff --git a/bu_isciii/templates/exometrio/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog b/bu_isciii/templates/exometrio/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog index 063393c2..e771e6dc 100644 --- a/bu_isciii/templates/exometrio/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog +++ b/bu_isciii/templates/exometrio/ANALYSIS/ANALYSIS01_EXOME/03-annotation/lablog @@ -28,7 +28,7 @@ echo "srun --partition short_idx --time 2:00:00 --chdir ${scratch_dir} --output # 3. Lablog for annotating whole genome samples using Variant Effect Predictor (VEP). # Run Vep without the plugin columns -echo "srun --partition short_idx --mem 100G --time 12:00:00 --chdir ${scratch_dir} --output logs/VEP.log --job-name VEP singularity exec -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/ensembl-vep:103.1--pl5262h4a94de4_2 vep --fasta /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/human_g1k_v37.fasta -i ${scratch_dir}/vep/variants_fil_mod.vcf -o ${scratch_dir}/vep/vep_annot.vcf --cache --offline --dir_cache /data/bi/references/eukaria/homo_sapiens/cache_vep/ --everything --assembly GRCh37 --tab &" > _02_vep_annotation.sh +echo "srun --partition short_idx --mem 100G --time 12:00:00 --chdir ${scratch_dir} --output logs/VEP.log --job-name VEP singularity exec -B /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome -B /data/bi/references/eukaria/homo_sapiens/cache_vep -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/ensembl-vep:103.1--pl5262h4a94de4_2 vep --fasta /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/human_g1k_v37.fasta -i ${scratch_dir}/vep/variants_fil_mod.vcf -o ${scratch_dir}/vep/vep_annot.vcf --cache --offline --dir_cache /data/bi/references/eukaria/homo_sapiens/cache_vep/ --everything --assembly GRCh37 --tab &" > _02_vep_annotation.sh #-------------------------------------------------------------------------------------------------------------------- diff --git a/bu_isciii/templates/wgstrio/ANALYSIS/ANALYSIS01_GENOME/03-annotation/lablog b/bu_isciii/templates/wgstrio/ANALYSIS/ANALYSIS01_GENOME/03-annotation/lablog index 50d87f79..c6a964bd 100644 --- a/bu_isciii/templates/wgstrio/ANALYSIS/ANALYSIS01_GENOME/03-annotation/lablog +++ b/bu_isciii/templates/wgstrio/ANALYSIS/ANALYSIS01_GENOME/03-annotation/lablog @@ -31,7 +31,7 @@ echo "srun --partition short_idx --time 2:00:00 --chdir ${scratch_dir} --output # Run Vep without the plugin columns -echo "srun --partition short_idx --mem 100G --time 12:00:00 --chdir ${scratch_dir} --output logs/VEP.log --job-name VEP singularity exec -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/ensembl-vep:103.1--pl5262h4a94de4_2 vep --fasta /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/human_g1k_v37.fasta -i ${scratch_dir}/vep/variants_fil_mod.vcf -o ${scratch_dir}/vep/vep_annot.vcf --cache --offline --dir_cache /data/bi/references/eukaria/homo_sapiens/cache_vep/ --everything --assembly GRCh37 --tab &" > _02_vep_annotation.sh +echo "srun --partition short_idx --mem 100G --time 12:00:00 --chdir ${scratch_dir} --output logs/VEP.log --job-name VEP singularity exec -B /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/ -B /data/bi/references/eukaria/homo_sapiens/cache_vep/ -B ${scratch_dir}/../../../ /data/bi/pipelines/singularity-images/ensembl-vep:103.1--pl5262h4a94de4_2 vep --fasta /data/bi/references/eukaria/homo_sapiens/hg19/1000genomes_b37/genome/human_g1k_v37.fasta -i ${scratch_dir}/vep/variants_fil_mod.vcf -o ${scratch_dir}/vep/vep_annot.vcf --cache --offline --dir_cache /data/bi/references/eukaria/homo_sapiens/cache_vep/ --everything --assembly GRCh37 --tab &" > _02_vep_annotation.sh #-------------------------------------------------------------------------------------------------------------------- From 80483d9cc7b1fb831c35cf79bd0d0de815fd204a Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 30 Sep 2024 16:50:49 +0200 Subject: [PATCH 2/9] Updated bioinfo_doc.py to allow more options when creating the delivery email --- bu_isciii/bioinfo_doc.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) mode change 100755 => 100644 bu_isciii/bioinfo_doc.py diff --git a/bu_isciii/bioinfo_doc.py b/bu_isciii/bioinfo_doc.py old mode 100755 new mode 100644 index 4da84672..458dd899 --- a/bu_isciii/bioinfo_doc.py +++ b/bu_isciii/bioinfo_doc.py @@ -646,10 +646,34 @@ def email_creation(self): email_data["email_notes"] = self.delivery_notes.replace( "\n", "
" ) + else: + email_data["email_notes"] = bu_isciii.utils.ask_for_some_text( + msg="Write email notes" + ).replace("\n", "
") else: - email_data["email_notes"] = bu_isciii.utils.ask_for_some_text( - msg="Write email notes" - ).replace("\n", "
") + if bu_isciii.utils.prompt_yn_question( + msg="Do you wish to provide a text file for email notes?", dflt=False + ): + for i in range(3, -1, -1): + email_data["email_notes"] = bu_isciii.utils.prompt_path( + msg="Write the path to the file with RAW text as email notes" + ) + if not os.path.isfile(os.path.expanduser(email_data["email_notes"])): + stderr.print(f"Provided file doesn't exist. Attempts left: {i}") + else: + stderr.print(f"File selected: {email_data["email_notes"]}") + break + else: + stderr.print("No more attempts. Email notes will be given by prompt") + email_data["email_notes"] = None + else: + email_data["email_notes"] = None + + if email_data["email_notes"]: + with open(os.path.expanduser(email_data["email_notes"])) as f: + email_data["email_notes"] = f.read().replace("\n", "
") + else: + email_data["email_notes"] = bu_isciii.utils.ask_for_some_text(msg="Write email notes").replace("\n", "
") email_data["user_data"] = self.resolution_info["service_user_id"] email_data["service_id"] = self.service_name.split("_", 5)[0] From 1b70cf262fef71fa7f7f822372a0959f4d93f0ce Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 30 Sep 2024 16:54:57 +0200 Subject: [PATCH 3/9] Updated excel_generator.py to be more informative --- .../viralrecon/RESULTS/excel_generator.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py b/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py index 8467c414..ffeece39 100755 --- a/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py +++ b/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py @@ -68,7 +68,7 @@ def merge_lineage_tables( csvs_in_folder=csvs_in_folder, merged_csv_name=merged_csv_name ) else: - print(f"No pangolin folder could be found for {ref}, omitting") + print(f"\033[93mNo pangolin folder could be found for {ref}, omitting\033[0m") if os.path.isdir(os.path.abspath(folder + "/nextclade")): nextcl_dir = os.path.join(folder, "nextclade") @@ -82,17 +82,18 @@ def merge_lineage_tables( csvs_in_folder=csvs_in_folder, merged_csv_name=merged_csv_name ) else: - print(f"No nextclade folder could be found for {ref}, omitting") + print(f"\033[93mNo nextclade folder could be found for {ref}, omitting\033[0m\n") return def excel_generator(csv_files: List[str]): + #print("Proceeding") for file in csv_files: if not os.path.exists(file): - print(f"File {file} does not exist, omitting...") + print(f"\033[91mFile {file} does not exist, omitting...\033[0m") continue - print(f"Generating excel file for {file}") + print(f"\033[92mGenerating excel file for {file}\033[0m") output_name = os.path.splitext(os.path.basename(file))[0] + ".xlsx" # workbook = openpyxl.Workbook(output_name) if "nextclade" in str(file): @@ -108,7 +109,7 @@ def excel_generator(csv_files: List[str]): try: table = pd.read_csv(file) except pd.errors.EmptyDataError: - print("Could not parse table from ", str(file)) + print("\033[91mCould not parse table from ", str(file), "\033[0m") continue table = table.drop(["index"], axis=1, errors="ignore") table.to_excel(output_name, index=False) @@ -119,22 +120,22 @@ def single_csv_to_excel(csv_file: str): try: excel_generator([csv_file]) except FileNotFoundError as e: - print(f"Could not find file {e}") + print(f"\033[91mCould not find file {e}\033[0m") def main(args): if args.single_csv: # If single_csv is called, just convert target csv to excel and skip the rest - print("Single file convertion selected. Skipping main process...") + print("\033[92mSingle file convertion selected. Skipping main process...\033[0m") single_csv_to_excel(args.single_csv) exit(0) print( - "Extracting references used for analysis and the samples associated with each reference\n" + "\033[92mExtracting references used for analysis and the samples associated with each reference\033[0m" ) with open(args.reference_file, "r") as file: references = [line.rstrip() for line in file] - print(f"\nFound {len(references)} references: {str(references).strip('[]')}") + print(f"\n\033[92mFound {len(references)} references: {str(references).strip('[]')}\033[0m") reference_folders = {ref: str("excel_files_" + ref) for ref in references} samples_ref_files = { @@ -145,7 +146,7 @@ def main(args): # Merge pangolin and nextclade csv files separatedly and create excel files for them merge_lineage_tables(reference_folders, samples_ref_files) for reference, folder in reference_folders.items(): - print(f"Creating excel files for reference {reference}") + print(f"\033[92mCreating excel files for reference {reference}\033[0m") csv_files = [ file.path for file in os.scandir(folder) if file.path.endswith(".csv") ] @@ -160,8 +161,9 @@ def main(args): csvs_in_folder=variants_tables, merged_csv_name="variants_long_table.csv" ) except FileNotFoundError as e: - print(str(e)) - print("Merged variants_long_table.csv might be empty") + print("\033[93mWARNING!\033[0m") + print("\033[93mAt least one variants_long_table.csv file could not be found. Therefore, merged variants_long_table.csv will be incomplete.\033[0m") + print("\033[93mPlease, check the following report in order to know which links are broken and, therefore, which tables could not be found:\033[0m\n") # Create excel files for individual tables valid_extensions = [".csv", ".tsv", ".tab"] @@ -173,6 +175,12 @@ def main(args): link_csvs = [file for file in rest_of_csvs if os.path.islink(file)] broken_links = [file for file in link_csvs if not os.path.exists(os.readlink(file))] valid_csvs = [file for file in rest_of_csvs if file not in broken_links] + + if broken_links: + print(f"\033[93mWARNING! {len(broken_links)} broken links found (for .csv, .tsv or .tab files). Please fix them.\033[0m") + for broken_link in broken_links: + print(f"\033[93mBroken link: {broken_link} (target: {os.readlink(broken_link)})\033[0m") + excel_generator(valid_csvs) From e4ceba01860782be83f4c7cfb82f20b04fc401d0 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 30 Sep 2024 16:55:21 +0200 Subject: [PATCH 4/9] Updated sftp_user.json --- bu_isciii/templates/sftp_user.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bu_isciii/templates/sftp_user.json b/bu_isciii/templates/sftp_user.json index bc7f4528..344c8b49 100755 --- a/bu_isciii/templates/sftp_user.json +++ b/bu_isciii/templates/sftp_user.json @@ -44,7 +44,7 @@ "sara.perez": ["GeneticDiagnosis"], "sbarturen": ["Labvirushep"], "sergio.sanchez": ["LabFWDB_ssanchez"], - "sherrera": ["LabFWBD", "LabFWBD_ext"], + "sherrera": ["LabFWBD", "LabFWBD_ext", "Labtuberculosis"], "sresino": ["Labvirushep"], "svaldezate": ["Labtaxonomia"], "svazquez": ["Labvirusres"], From 75ad889f9575016a4cacaa4d44a7ea9889f508da Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 30 Sep 2024 16:56:29 +0200 Subject: [PATCH 5/9] Updated 03-MTBSeq/lablog to handle the first sample name appropriately --- .../mtbseq/ANALYSIS/ANALYSIS01_MTBSEQ/03-MTBSeq/lablog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bu_isciii/templates/mtbseq/ANALYSIS/ANALYSIS01_MTBSEQ/03-MTBSeq/lablog b/bu_isciii/templates/mtbseq/ANALYSIS/ANALYSIS01_MTBSEQ/03-MTBSeq/lablog index e5bdac16..0eff7900 100644 --- a/bu_isciii/templates/mtbseq/ANALYSIS/ANALYSIS01_MTBSEQ/03-MTBSeq/lablog +++ b/bu_isciii/templates/mtbseq/ANALYSIS/ANALYSIS01_MTBSEQ/03-MTBSeq/lablog @@ -13,9 +13,9 @@ cat ../samples_id.txt | xargs -I @@ echo -e "srun --job-name MTBSEQ.@@ --output # classification echo "mkdir classification_all" > _03_gather_results.sh -echo "FIRST_SAMPLE=$(head -n1 ../samples_id.txt); head -n 1 ${FIRST_SAMPLE}/Classification/Strain_Classification.tab > classification_all/strain_classification_all.tab; grep \"^'$analysis_year\" */Classification/Strain_Classification.tab | cut -d \":\" -f 2 >> classification_all/strain_classification_all.tab" >> _03_gather_results.sh +echo 'FIRST_SAMPLE=$(head -n1 samples_id.txt); head -n 1 ${FIRST_SAMPLE}/Classification/Strain_Classification.tab > classification_all/strain_classification_all.tab; grep "^'\'''"$analysis_year"'" */Classification/Strain_Classification.tab | cut -d ":" -f 2 >> classification_all/strain_classification_all.tab' >> _03_gather_results.sh # resistances echo "mkdir resistances_all" >> _03_gather_results.sh cat ../samples_id.txt | xargs -I % echo "cp %/Amend/NONE_joint_cf4_cr4_fr75_ph4_samples1_amended.tab resistances_all/%_var_res.tab" >> _03_gather_results.sh # stats -echo "mkdir stats_all; FIRST_SAMPLE=$(head -n1 ../samples_id.txt); head -n 1 ${FIRST_SAMPLE}/Statistics/Mapping_and_Variant_Statistics.tab > stats_all/statistics_all.tab; grep \"^'$analysis_year\" */Statistics/Mapping_and_Variant_Statistics.tab | cut -d \":\" -f 2 >> stats_all/statistics_all.tab" >> _03_gather_results.sh +echo 'mkdir stats_all; FIRST_SAMPLE=$(head -n1 ../samples_id.txt); head -n 1 ${FIRST_SAMPLE}/Statistics/Mapping_and_Variant_Statistics.tab > stats_all/statistics_all.tab; grep "^'\'''"$analysis_year"'" */Statistics/Mapping_and_Variant_Statistics.tab | cut -d ":" -f 2 >> stats_all/statistics_all.tab' >> _03_gather_results.sh From 04c7ac67f82f574733992d44560e8ed2b690bddd Mon Sep 17 00:00:00 2001 From: victor5lm Date: Mon, 30 Sep 2024 17:16:18 +0200 Subject: [PATCH 6/9] Fixed linting mistakes --- bu_isciii/bioinfo_doc.py | 21 +++++++--- .../viralrecon/RESULTS/excel_generator.py | 38 +++++++++++++------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/bu_isciii/bioinfo_doc.py b/bu_isciii/bioinfo_doc.py index 458dd899..519cb382 100644 --- a/bu_isciii/bioinfo_doc.py +++ b/bu_isciii/bioinfo_doc.py @@ -652,19 +652,26 @@ def email_creation(self): ).replace("\n", "
") else: if bu_isciii.utils.prompt_yn_question( - msg="Do you wish to provide a text file for email notes?", dflt=False + msg="Do you wish to provide a text file for email notes?", + dflt=False, ): for i in range(3, -1, -1): email_data["email_notes"] = bu_isciii.utils.prompt_path( msg="Write the path to the file with RAW text as email notes" ) - if not os.path.isfile(os.path.expanduser(email_data["email_notes"])): - stderr.print(f"Provided file doesn't exist. Attempts left: {i}") + if not os.path.isfile( + os.path.expanduser(email_data["email_notes"]) + ): + stderr.print( + f"Provided file doesn't exist. Attempts left: {i}" + ) else: - stderr.print(f"File selected: {email_data["email_notes"]}") + stderr.print(f"File selected: {email_data['email_notes']}") break else: - stderr.print("No more attempts. Email notes will be given by prompt") + stderr.print( + "No more attempts. Email notes will be given by prompt" + ) email_data["email_notes"] = None else: email_data["email_notes"] = None @@ -673,7 +680,9 @@ def email_creation(self): with open(os.path.expanduser(email_data["email_notes"])) as f: email_data["email_notes"] = f.read().replace("\n", "
") else: - email_data["email_notes"] = bu_isciii.utils.ask_for_some_text(msg="Write email notes").replace("\n", "
") + email_data["email_notes"] = bu_isciii.utils.ask_for_some_text( + msg="Write email notes" + ).replace("\n", "
") email_data["user_data"] = self.resolution_info["service_user_id"] email_data["service_id"] = self.service_name.split("_", 5)[0] diff --git a/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py b/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py index ffeece39..2cdb83b4 100755 --- a/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py +++ b/bu_isciii/templates/viralrecon/RESULTS/excel_generator.py @@ -68,7 +68,9 @@ def merge_lineage_tables( csvs_in_folder=csvs_in_folder, merged_csv_name=merged_csv_name ) else: - print(f"\033[93mNo pangolin folder could be found for {ref}, omitting\033[0m") + print( + f"\033[93mNo pangolin folder could be found for {ref}, omitting\033[0m" + ) if os.path.isdir(os.path.abspath(folder + "/nextclade")): nextcl_dir = os.path.join(folder, "nextclade") @@ -82,13 +84,15 @@ def merge_lineage_tables( csvs_in_folder=csvs_in_folder, merged_csv_name=merged_csv_name ) else: - print(f"\033[93mNo nextclade folder could be found for {ref}, omitting\033[0m\n") + print( + f"\033[93mNo nextclade folder could be found for {ref}, omitting\033[0m\n" + ) return def excel_generator(csv_files: List[str]): - #print("Proceeding") + # print("Proceeding") for file in csv_files: if not os.path.exists(file): print(f"\033[91mFile {file} does not exist, omitting...\033[0m") @@ -126,7 +130,9 @@ def single_csv_to_excel(csv_file: str): def main(args): if args.single_csv: # If single_csv is called, just convert target csv to excel and skip the rest - print("\033[92mSingle file convertion selected. Skipping main process...\033[0m") + print( + "\033[92mSingle file convertion selected. Skipping main process...\033[0m" + ) single_csv_to_excel(args.single_csv) exit(0) @@ -135,7 +141,9 @@ def main(args): ) with open(args.reference_file, "r") as file: references = [line.rstrip() for line in file] - print(f"\n\033[92mFound {len(references)} references: {str(references).strip('[]')}\033[0m") + print( + f"\n\033[92mFound {len(references)} references: {str(references).strip('[]')}\033[0m" + ) reference_folders = {ref: str("excel_files_" + ref) for ref in references} samples_ref_files = { @@ -160,10 +168,14 @@ def main(args): concat_tables_and_write( csvs_in_folder=variants_tables, merged_csv_name="variants_long_table.csv" ) - except FileNotFoundError as e: + except FileNotFoundError: print("\033[93mWARNING!\033[0m") - print("\033[93mAt least one variants_long_table.csv file could not be found. Therefore, merged variants_long_table.csv will be incomplete.\033[0m") - print("\033[93mPlease, check the following report in order to know which links are broken and, therefore, which tables could not be found:\033[0m\n") + print( + "\033[93mAt least one variants_long_table.csv file could not be found. Therefore, merged variants_long_table.csv will be incomplete.\033[0m" + ) + print( + "\033[93mPlease, check the following report in order to know which links are broken and, therefore, which tables could not be found:\033[0m\n" + ) # Create excel files for individual tables valid_extensions = [".csv", ".tsv", ".tab"] @@ -177,9 +189,13 @@ def main(args): valid_csvs = [file for file in rest_of_csvs if file not in broken_links] if broken_links: - print(f"\033[93mWARNING! {len(broken_links)} broken links found (for .csv, .tsv or .tab files). Please fix them.\033[0m") - for broken_link in broken_links: - print(f"\033[93mBroken link: {broken_link} (target: {os.readlink(broken_link)})\033[0m") + print( + f"\033[93mWARNING! {len(broken_links)} broken links found (for .csv, .tsv or .tab files). Please fix them.\033[0m" + ) + for broken_link in broken_links: + print( + f"\033[93mBroken link: {broken_link} (target: {os.readlink(broken_link)})\033[0m" + ) excel_generator(valid_csvs) From de1f4716208d29e5f9a8b3962242a65bd019b87b Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 1 Oct 2024 12:05:50 +0200 Subject: [PATCH 7/9] Updated CHANGELOG.md --- CHANGELOG.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b7e3f90..b76da9b9 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,15 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), Code contributions to the new version: -- [Daniel Valle](https://github.com/Daniel-VM) -- [Sarai Varona](https://github.com/svarona) - ### Template fixes and updates -- Fixed path to blast database and update Emmtyper params [#339](https://github.com/BU-ISCIII/buisciii-tools/pull/339) -- Updated sarek version (v3.4.4) in ExomeEB-ExomeTrio-WGSTrio templates [#341] (https://github.com/BU-ISCIII/buisciii-tools/pull/341) -- Fixed IRMAs config for amended consensus [#325](https://github.com/BU-ISCIII/buisciii-tools/pull/325). - ### Modules #### Added enhancements @@ -51,6 +44,36 @@ Code contributions to the hotfix: ### Requirements +## [2.2.1] - 2024-10-01 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.1 + +### Credits + +Code contributions to the new version: + +- [Daniel Valle](https://github.com/Daniel-VM) +- [Sarai Varona](https://github.com/svarona) +- [Victor Lopez](https://github.com/victor5lm) +- [Sergio Olmos](https://github.com/OPSergio) + +### Template fixes and updates + +- Fixed path to blast database and update Emmtyper params [#339](https://github.com/BU-ISCIII/buisciii-tools/pull/339) +- Updated sarek version (v3.4.4) in ExomeEB-ExomeTrio-WGSTrio templates [#341] (https://github.com/BU-ISCIII/buisciii-tools/pull/341) +- Fixed IRMAs config for amended consensus [#325](https://github.com/BU-ISCIII/buisciii-tools/pull/325). +- Improved excel_generator.py and bioinfo_doc.py email creation function, and updated sftp_user.json, setup.py, main.py and some lablogs [#344](https://github.com/BU-ISCIII/buisciii-tools/pull/344). + +### Modules + +#### Added enhancements + +#### Fixes + +#### Changed + +#### Removed + +### Requirements + ## [2.2.0] - 2024-09-12 : https://github.com/BU-ISCIII/buisciii-tools/releases/tag/2.2.0 ### Credits From 74591d6c2f2eb70fb158e66ac091b2fe6a7e06d6 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 1 Oct 2024 12:06:07 +0200 Subject: [PATCH 8/9] Updated setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index efa93a59..50fdff3d 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.2.0" +version = "2.2.1" with open("README.md") as f: readme = f.read() From d7ba96e8a34bea0a3e8281450b6836d42b1fe69f Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 1 Oct 2024 12:06:22 +0200 Subject: [PATCH 9/9] Updated __main__.py --- bu_isciii/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bu_isciii/__main__.py b/bu_isciii/__main__.py index 3933a027..1aebd780 100755 --- a/bu_isciii/__main__.py +++ b/bu_isciii/__main__.py @@ -57,7 +57,7 @@ def run_bu_isciii(): ) # stderr.print("[green] `._,._,'\n", highlight=False) - __version__ = "2.2.0" + __version__ = "2.2.1" stderr.print( "[grey39] BU-ISCIII-tools version {}".format(__version__), highlight=False )