From bcd9a87d788c81efa09b062cc4f51a8c1a9feebe Mon Sep 17 00:00:00 2001 From: xonq Date: Fri, 17 Jul 2026 23:18:40 +0000 Subject: [PATCH 01/34] refactor --- mycotools/acc2fa.py | 23 +- mycotools/acc2fq.py | 7 +- mycotools/acc2gbk.py | 11 +- mycotools/acc2gff.py | 14 +- mycotools/acc2locus.py | 13 +- mycotools/add2gff.py | 46 +-- mycotools/annotationStats.py | 12 +- mycotools/assemblyStats.py | 16 +- mycotools/coords2fa.py | 21 +- mycotools/crap.py | 277 ++++++++--------- mycotools/db2files.py | 55 ++-- mycotools/db2hgs.py | 82 ++--- mycotools/db2microsyntree.py | 106 ++++--- mycotools/db2search.py | 179 +++++------ mycotools/extract_mtdb.py | 140 ++------- mycotools/fa2clus.py | 126 ++++---- mycotools/fa2hmmer2fa.py | 37 ++- mycotools/fa2mass.py | 8 +- mycotools/fa2tree.py | 206 ++++++------- mycotools/fna2faa.py | 7 +- mycotools/gff2seq.py | 14 +- mycotools/gff2svg.py | 33 +- mycotools/jgiDwnld.py | 157 +++++----- mycotools/lib/biotools.py | 1 - mycotools/lib/dbtools.py | 450 +++++++++++++++++---------- mycotools/lib/kontools.py | 248 ++++++++------- mycotools/manage_mtdb.py | 19 +- mycotools/mtdb.py | 309 +++++++++---------- mycotools/ncbiAcc2fa.py | 16 +- mycotools/ncbiDwnld.py | 93 +++--- mycotools/ncbi_dwnld_fallback.py | 123 ++++---- mycotools/ome2name.py | 16 +- mycotools/predb2mtdb.py | 155 ++++------ mycotools/treetools.py | 28 +- mycotools/update_mtdb.py | 456 +++++++++++++--------------- mycotools/utils/curGFF3.py | 8 +- mycotools/utils/extractHmmAcc.py | 26 +- mycotools/utils/extractHmmsearch.py | 33 +- mycotools/utils/gff2gff3.py | 25 +- mycotools/utils/gtf2gff3.py | 28 +- mycotools/utils/jgi2db.py | 67 ++-- mycotools/utils/ncbi2db.py | 18 +- mycotools/utils/og2mycodb.py | 4 +- pyproject.toml | 24 +- 44 files changed, 1858 insertions(+), 1879 deletions(-) diff --git a/mycotools/acc2fa.py b/mycotools/acc2fa.py index 8167a4d..bd39391 100755 --- a/mycotools/acc2fa.py +++ b/mycotools/acc2fa.py @@ -1,13 +1,15 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import argparse from collections import defaultdict from mycotools.lib.biotools import fa2dict, dict2fa, reverse_complement from mycotools.lib.dbtools import mtdb, primaryDB -from mycotools.lib.kontools import format_path, eprint, stdin2str +from mycotools.lib.kontools import format_path, stdin2str, setup_logging + +logger = logging.getLogger(__name__) def extract_mtdb_accs_exp(fa_dict, accs): @@ -34,7 +36,7 @@ def extract_mtdb_accs(fa_dict, accs, spacer=""): try: out_fa[acc] = fa_dict[acc] except KeyError: - eprint(f"WARNING: {acc} has no CDS", flush=True) + logger.warning(f"{acc} has no CDS") continue acc_name = acc[: acc.find("[")] if start < end: @@ -44,9 +46,7 @@ def extract_mtdb_accs(fa_dict, accs, spacer=""): "description": fa_dict[acc_name]["description"], } except KeyError: - eprint( - spacer + "WARNING: invalid accession " + acc_name, flush=True - ) + logger.warning(spacer + "invalid accession " + acc_name) else: try: out_fa[acc] = { @@ -56,14 +56,12 @@ def extract_mtdb_accs(fa_dict, accs, spacer=""): "description": fa_dict[acc_name]["description"], } except KeyError: - eprint( - spacer + "WARNING: invalid accession " + acc_name, flush=True - ) + logger.warning(spacer + "invalid accession " + acc_name) else: # no coordinates try: out_fa[acc] = fa_dict[acc] # extract the whole accession except KeyError: - eprint(spacer + "WARNING: invalid accession " + acc_name, flush=True) + logger.warning(spacer + "invalid accession " + acc_name) return out_fa @@ -138,7 +136,7 @@ def dbmain(db, accs, error=True, spacer="\t\t\t", coord_check=True): if error: raise KeyError("invalid ome: " + ome) else: - eprint(spacer + ome + " not in database", flush=True) + logger.info(spacer + ome + " not in database") fa_dict = {**fa_dict, **extract_mtdb_accs(ome_fasta, ome_accs)} else: for ome, ome_accs in ome_data.items(): @@ -148,7 +146,7 @@ def dbmain(db, accs, error=True, spacer="\t\t\t", coord_check=True): if error: raise KeyError else: - eprint(spacer + ome + " not in database", flush=True) + logger.info(spacer + ome + " not in database") fa_dict = {**fa_dict, **extract_mtdb_accs_exp(ome_fasta, ome_accs)} return fa_dict @@ -187,6 +185,7 @@ def cli(): parser.add_argument("-e", "--end", help="End index column (1 indexed)", type=int) parser.add_argument("-d", "--mtdb", default=primaryDB()) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.input: # input file input_file = format_path(args.input) diff --git a/mycotools/acc2fq.py b/mycotools/acc2fq.py index 1cd278f..4e06fe6 100755 --- a/mycotools/acc2fq.py +++ b/mycotools/acc2fq.py @@ -1,6 +1,6 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import gzip @@ -9,7 +9,9 @@ from collections import defaultdict # from mycotools.lib.biotools import dict2fq -from mycotools.lib.kontools import format_path, eprint, stdin2str +from mycotools.lib.kontools import format_path, stdin2str, setup_logging + +logger = logging.getLogger(__name__) def dict2fq(fastq_dict, description=True): @@ -189,6 +191,7 @@ def cli(): parser.add_argument("-i", "--input", help="File with accessions") parser.add_argument("-f", "--fastq", help="FASTQ input", required=True) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.input: # input file input_file = format_path(args.input) diff --git a/mycotools/acc2gbk.py b/mycotools/acc2gbk.py index c228ee1..0345863 100755 --- a/mycotools/acc2gbk.py +++ b/mycotools/acc2gbk.py @@ -1,17 +1,19 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import argparse import multiprocessing as mp from itertools import chain from collections import defaultdict -from mycotools.lib.kontools import eprint, format_path, stdin2str +from mycotools.lib.kontools import format_path, stdin2str, setup_logging from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.biotools import fa2dict, gff2list, gff3Comps from mycotools.acc2gff import db_main as acc2gff +logger = logging.getLogger(__name__) + def col_CDS( gff_list, types={"gene", "CDS", "exon", "mRNA", "tRNA", "rRNA", "RNA", "pseudogene"} @@ -28,7 +30,7 @@ def col_CDS( try: alias = re.search(gff3Comps()["Alias"], entry["attributes"])[1] except TypeError: - eprint("\n\tERROR: could not extract Alias ID from " + gff, flush=True) + logger.error("could not extract Alias ID from " + gff) continue aliases = alias.split("|") # to address alternate splicing in gene # aliases @@ -535,6 +537,7 @@ def cli(): parser.add_argument("-d", "--mtdb", help="DEFAULT: master", default=primaryDB()) parser.add_argument("-c", "--cpu", type=int, default=1) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) # gather accessions from various input styles and split into a list if args.input: @@ -566,7 +569,7 @@ def cli(): # only full genomes for args.full, no accessions if args.full: if any("_" in x for x in accs): # _ is forbidden from ome codes - eprint("\nERROR: -f requires omes, not accessions", flush=True) + logger.error("-f requires omes, not accessions") sys.exit(3) # create a default regular expression for the product name diff --git a/mycotools/acc2gff.py b/mycotools/acc2gff.py index fefecfa..f234033 100755 --- a/mycotools/acc2gff.py +++ b/mycotools/acc2gff.py @@ -1,13 +1,16 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import argparse import multiprocessing as mp from mycotools.lib.biotools import gff2list, list2gff from mycotools.lib.dbtools import mtdb, primaryDB -from mycotools.lib.kontools import format_path, stdin2str +from mycotools.lib.kontools import format_path, stdin2str, setup_logging +from pathlib import Path + +logger = logging.getLogger(__name__) def grab_gff_acc(gff_list, acc, term="Alias="): @@ -107,6 +110,7 @@ def cli(): ) parser.add_argument("--cpu", type=int, default=mp.cpu_count()) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) # if there is an input file, extract the accessions from that if args.input: @@ -151,13 +155,13 @@ def cli(): print(list2gff(gff_lists[list(gff_lists.keys())[0]]).rstrip(), flush=True) # if it is specified output, open a folder for it elif args.ome: - output = mkOutput(os.getcwd() + "/", "acc2gff") + output = mkOutput(str(Path.cwd()) + "/", "acc2gff") for ome in gff_strs: if gff_lists[ome]: with open(output + ome + ".accs.gff3", "w") as out: out.write(list2gff(gff_lists[ome])) else: - eprint("ERROR: " + ome + " failed, no accessions retrieved", flush=True) + logger.error("" + ome + " failed, no accessions retrieved") # print to stdout each gff_list else: out_str = "" @@ -165,7 +169,7 @@ def cli(): if gff_lists[ome]: out_str += list2gff(gff_lists[ome]) + "\n" else: - eprint("ERROR: " + ome + " does not have accession", flush=True) + logger.error("" + ome + " does not have accession") print(out_str) sys.exit(0) diff --git a/mycotools/acc2locus.py b/mycotools/acc2locus.py index 16b380f..39d3cee 100755 --- a/mycotools/acc2locus.py +++ b/mycotools/acc2locus.py @@ -1,17 +1,19 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import argparse import multiprocessing as mp from itertools import chain from collections import defaultdict -from mycotools.lib.kontools import eprint, format_path, file2list, stdin2str +from mycotools.lib.kontools import format_path, file2list, stdin2str, setup_logging from mycotools.lib.dbtools import primaryDB, mtdb from mycotools.lib.biotools import gff2list, fa2dict, dict2fa, list2gff, gff3Comps from mycotools.acc2gff import grab_gff_acc +logger = logging.getLogger(__name__) + def prep_gff_output(hit_list, gff_path, cpu=1): """Prepare an output file for gffs""" @@ -262,6 +264,7 @@ def cli(): ) parser.add_argument("--cpu", type=int, default=1) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.cpu < mp.cpu_count(): cpu = args.cpu @@ -284,15 +287,15 @@ def cli(): else: accs = [args.acc] else: - eprint("\nERROR: requires input or acc", flush=True) + logger.error("requires input or acc") sys.exit(1) if args.between: if len(accs) > 2: - eprint("\nERROR: -b needs 2 accessions", flush=True) + logger.error("-b needs 2 accessions") sys.exit(2) if args.nucleotide: - eprint("\nERROR: -b and -n are incompatible", flush=True) + logger.error("-b and -n are incompatible") sys.exit(3) db = None diff --git a/mycotools/add2gff.py b/mycotools/add2gff.py index ef00430..0be104d 100755 --- a/mycotools/add2gff.py +++ b/mycotools/add2gff.py @@ -7,15 +7,18 @@ # NEED a protein_id import option -import os +import logging import re import sys import argparse from collections import defaultdict -from mycotools.lib.kontools import sys_start, format_path, eprint, mkOutput +from mycotools.lib.kontools import sys_start, format_path, mkOutput, setup_logging from mycotools.lib.biotools import gff2list, list2gff, gff3Comps, gff2Comps, gtfComps from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.utils.curGFF3 import rename_and_organize +from pathlib import Path + +logger = logging.getLogger(__name__) def determine_version(toadd_gff, ome=None): @@ -32,7 +35,7 @@ def determine_version(toadd_gff, ome=None): from mycotools.utils.gtf2gff3 import main as curAnn if not ome: - eprint("\nOme required for gtf input", flush=True) + logger.info("Ome required for gtf input") sys.exit(1) new_gff = curAnn(toadd_gff, ome)[0] return new_gff, gff3Comps() @@ -100,7 +103,7 @@ def parse_toadd(toadd_gff, comps, ome, mtdb_acc=0): score, phase = gene["score"], gene["phase"] gene_acc = ome + "_manual" + str(mtdb_acc) if len(gene_dict["rna"]) > 1: - eprint("\nAlternately spliced loci currently not supported") + logger.info("Alternately spliced loci currently not supported") sys.exit(2) gene_dict["cds"] = sorted(gene_dict["cds"], key=lambda x: x[0]) @@ -229,7 +232,7 @@ def add_to_mtdb_gff(curadd_gff, addto_gff, ome, replace=False): ref_acc ] = new_acc # what about multiple gene # updates? e.g. fusions? this would delete - eprint(ref_acc + "\t->\t" + new_acc, flush=True) + logger.info(ref_acc + "->\t" + new_acc) out_gff = [] for entry in addto_gff: @@ -262,16 +265,16 @@ def prep_mtdb_update(new_gff, ome, db): from mycotools.predb2mtdb import main as predb2mtdb from mycotools.lib.dbtools import mtdb, primaryDB - out_dir = mkOutput(format_path(os.getcwd()), "add2gff") + out_dir = mkOutput(format_path(str(Path.cwd())), "add2gff") wrk_dir = out_dir + "working/" - if not os.path.isdir(wrk_dir): - os.mkdir(out_dir + "working/") - if not os.path.isdir(wrk_dir + "fna/"): - os.mkdir(out_dir + "working/fna/") - if not os.path.isdir(wrk_dir + "gff3/"): - os.mkdir(out_dir + "working/gff3/") - if not os.path.isdir(wrk_dir + "faa/"): - os.mkdir(out_dir + "working/faa/") + if not Path(wrk_dir).is_dir(): + Path(out_dir + "working/").mkdir() + if not Path(wrk_dir + "fna/").is_dir(): + Path(out_dir + "working/fna/").mkdir() + if not Path(wrk_dir + "gff3/").is_dir(): + Path(out_dir + "working/gff3/").mkdir() + if not Path(wrk_dir + "faa/").is_dir(): + Path(out_dir + "working/faa/").mkdir() new_gff_path = out_dir + "working/gff3/" + ome + ".new.gff3" with open(new_gff_path, "w") as out: @@ -298,7 +301,7 @@ def prep_mtdb_update(new_gff, ome, db): update_db_path = out_dir + "add2gff.mtdb" count = 1 - while os.path.isfile(update_db_path): + while Path(update_db_path).is_file(): update_db_path = re.sub(r"_\d+$", "", update_db_path) update_db_path += f"_{count}" count += 1 @@ -334,6 +337,7 @@ def cli(): ) parser.add_argument("-d", "--mtdb", default=primaryDB()) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) # usage = 'Add gff to an existing mtdb gff.\n' \ # + 'add2gff.py ' @@ -342,10 +346,10 @@ def cli(): if not args.addto: if args.update: - eprint("\nERROR: -u requires -a", flush=True) + logger.error("-u requires -a") sys.exit(7) if not args.ome: - eprint("\nERROR: need -a or -o", flush=True) + logger.error("need -a or -o") sys.exit(5) else: ome = args.ome @@ -353,14 +357,14 @@ def cli(): else: ome = None addto_file = format_path(args.addto) - if not os.path.isfile(addto_file): - eprint("\nERROR: -a does not exist", flush=True) + if not Path(addto_file).is_file(): + logger.error("-a does not exist") sys.exit(6) addto_gff = gff2list(addto_file) toadd_file = format_path(args.input) - if not os.path.isfile(toadd_file): - eprint("\nERROR: -i does not exist", flush=True) + if not Path(toadd_file).is_file(): + logger.error("-i does not exist") sys.exit(7) toadd_gff = import_toadd_gff(toadd_file) diff --git a/mycotools/annotationStats.py b/mycotools/annotationStats.py index bc7b246..0e16f3f 100755 --- a/mycotools/annotationStats.py +++ b/mycotools/annotationStats.py @@ -5,10 +5,15 @@ import os import re import sys +import logging from itertools import chain from collections import defaultdict from mycotools.lib.biotools import gff2list, gff3Comps -from mycotools.lib.kontools import format_path, eprint +from mycotools.lib.kontools import format_path, setup_logging +from pathlib import Path + + +logger = logging.getLogger(__name__) def compile_alia(gff_path, output, ome=None): @@ -250,7 +255,7 @@ def main(in_path, log_path=None, cpus=1, db=None): db = mtdb(in_path).set_index() prevOmes = {} - if log_path and os.path.isfile(log_path): + if log_path and Path(log_path).is_file(): with open(log_path, "r") as raw: for line in raw: if not line.startswith("#"): @@ -299,6 +304,7 @@ def main(in_path, log_path=None, cpus=1, db=None): def cli(): + setup_logging() output = False usage = "\nUSAGE: `gff`/`gtf`/`gff3` OR mycotoolsDB, optional output file\n" if "-h " in sys.argv or "--help" in sys.argv or "-h" == sys.argv[-1]: @@ -307,7 +313,7 @@ def cli(): elif len(sys.argv) < 2: print(usage, flush=True) sys.exit(1) - elif not os.path.isfile(format_path(sys.argv[1])): + elif not Path(format_path(sys.argv[1])).is_file(): print(usage, flush=True) sys.exit(1) elif len(sys.argv) > 2: diff --git a/mycotools/assemblyStats.py b/mycotools/assemblyStats.py index 90c0393..385d016 100755 --- a/mycotools/assemblyStats.py +++ b/mycotools/assemblyStats.py @@ -10,10 +10,15 @@ import os import sys import copy +import logging import multiprocessing as mp from mycotools.lib.dbtools import mtdb from mycotools.lib.biotools import fa2dict -from mycotools.lib.kontools import format_path, eprint +from mycotools.lib.kontools import format_path, setup_logging +from pathlib import Path + + +logger = logging.getLogger(__name__) def calcMask(contig_list): @@ -141,7 +146,7 @@ def main(in_path, log_path=None, cpus=1, db=None): # parse the output file if it currently exists to avoid redundant runs prev_omes = {} if log_path: - if not os.path.isfile(log_path): + if not Path(log_path).is_file(): with open(log_path, "w") as log_open: log_open.write(head) else: @@ -176,7 +181,7 @@ def main(in_path, log_path=None, cpus=1, db=None): if res[1]: calcs[res[0]] = "\t".join([str(x[1]) for x in res[1]]) else: - eprint("\t\tERROR:\t" + ome, flush=True) + logger.error(ome) # sort the results by the ome code alphabetically calcs = { @@ -202,9 +207,9 @@ def main(in_path, log_path=None, cpus=1, db=None): sortedContigs = sortContigs(in_path) calculations = n50l50(sortedContigs) if calculations: - stats[os.path.basename(os.path.abspath(in_path))] = n50l50(sortedContigs) + stats[Path(os.path.abspath(in_path)).name] = n50l50(sortedContigs) else: - eprint("\tERROR:\t" + in_path, flush=True) + logger.error(in_path) # print the stats to standard out, depending on if there are contigs # less than 1000 bp @@ -223,6 +228,7 @@ def main(in_path, log_path=None, cpus=1, db=None): def cli(): + setup_logging() usage = "\nUSAGE: assembly statistics\nAssembly `fasta` or mycotoolsDB, optional output file if using database\n" if {"-h", "--help"}.intersection(set(sys.argv)): print(usage, flush=True) diff --git a/mycotools/coords2fa.py b/mycotools/coords2fa.py index d266ee9..40d7448 100755 --- a/mycotools/coords2fa.py +++ b/mycotools/coords2fa.py @@ -2,13 +2,16 @@ # NEED to flag overlapping coordinates -import os +import logging import sys import argparse from Bio.Seq import Seq from collections import defaultdict from mycotools.lib.biotools import fa2dict, dict2fa -from mycotools.lib.kontools import sys_start, eprint, format_path +from mycotools.lib.kontools import sys_start, format_path +from pathlib import Path + +logger = logging.getLogger(__name__) def extractCoords(fa_dict, seqid, coord_start=0, coord_end=-1, sense="+", fa_name=""): @@ -67,9 +70,9 @@ def cli(): try: fa_file = format_path(args[0]) if fa_file.endswith("/"): # if it is a directory that was inputted - fa_name = os.path.basename(fa_file[:-1]) + fa_name = Path(fa_file[:-1]).name else: - fa_name = os.path.basename(fa_file) + fa_name = Path(fa_file).name fa = fa2dict(fa_file) @@ -91,7 +94,7 @@ def cli(): # print the extracted sequences to stdout in FASTA format print(dict2fa(out_fa)) - eprint(error) + logger.error(error) sys.exit(0) except IndexError: # fasta was not parseable @@ -125,7 +128,7 @@ def cli(): ) except: - eprint("\nERROR: incorrectly formatted input", flush=True) + logger.error("incorrectly formatted input") # for each file and coordinates, prepare for output for fa_file, concats in files_data.items(): @@ -133,9 +136,9 @@ def cli(): # extract the file name from the path if fa_file.endswith("/"): - fa_name = os.path.basename(fa_file[:-1]) + fa_name = Path(fa_file[:-1]).name else: - fa_name = os.path.basename(fa_file) + fa_name = Path(fa_file).name for concat_id, rows in concats.items(): if rows[0][-1] == "+": @@ -166,7 +169,7 @@ def cli(): out_fa = {**out_fa, **{fa_name + "_concat" + str(concat_id): toadd_fa}} print(dict2fa(out_fa)) - eprint(error) + logger.error(error) sys.exit(0) diff --git a/mycotools/crap.py b/mycotools/crap.py index 3ebb5b6..cffa286 100755 --- a/mycotools/crap.py +++ b/mycotools/crap.py @@ -15,6 +15,7 @@ # NEED assembly reference method, i.e. tblastn # NEED cluster variable input default +import logging import os import re import sys @@ -36,7 +37,6 @@ ) from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.kontools import ( - eprint, format_path, findExecs, intro, @@ -46,6 +46,7 @@ stdin2str, getColors, collect_files, + setup_logging, ) from mycotools.lib.biotools import fa2dict, dict2fa, gff2list, list2gff, gff3Comps from mycotools.acc2fa import dbmain as acc2fa @@ -64,6 +65,9 @@ # from mycotools.utils.og2mycodb import mycodbHGs, extract_ogs from mycotools.db2microsyntree import compile_homolog_groups +from pathlib import Path + +logger = logging.getLogger(__name__) os.environ["QT_QPA_PLATFORM"] = "offscreen" @@ -172,8 +176,8 @@ def input_genes2input_hgs(input_genes, gene2hg): try: input_hgs[gene] = gene2hg[gene] except KeyError: - eprint("ERROR: " + gene + " query with no HG", flush=True) - eprint("\t" + gene + " will be ignored.", flush=True) + logger.error("" + gene + " query with no HG") + logger.info("" + gene + " will be ignored.") return input_hgs @@ -189,11 +193,11 @@ def check_fa_size(fas, max_size): submitted to tree building""" fas4clus, fas4trees = {}, {} for query, fa in fas.items(): - print("\t" + query + "\t" + str(len(fa)) + " genes", flush=True) + logger.debug("" + query + "\t" + str(len(fa)) + " genes") if len(fa) > max_size: fas4clus[query] = fa elif len(fa) < 3: - eprint(f"\t\tWARNING: too few hits ({len(fa)})", flush=True) + logger.warning(f"too few hits ({len(fa)})") else: fas4trees[query] = fa @@ -208,13 +212,13 @@ def write_seq_clus(gene_module, focal_gene, db, output_path, out_fa): try: fa_dict = acc2fa(db, gene_module) except KeyError: - eprint("\t\t\tWARNING: some hits not in database", flush=True) + logger.warning("some hits not in database") db_omes = set(db.keys()) gene_module = [x for x in gene_module if x[: x.find("_")] in db_omes] fa_dict = acc2fa(db, gene_module) # need to implement some method to choose if the max and min parameters couldn't be met - print("\t\t\t" + str(len(fa_dict)) + " genes in group", flush=True) + logger.debug("" + str(len(fa_dict)) + " genes in group") with open(out_fa, "w") as out: out.write(dict2fa(fa_dict)) @@ -247,15 +251,15 @@ def run_fa2clus( # output Diamond data to a diamond directory dmnd_dir = output + "dmnd/" - if not os.path.isdir(dmnd_dir): - os.mkdir(dmnd_dir) + if not Path(dmnd_dir).is_dir(): + Path(dmnd_dir).mkdir() # create an output path and a log path for the inputted gene output_path = output + str(focal_gene) log_path = output + "." + str(focal_gene) + ".log" # parse an existing fa2clus_log - if os.path.isfile(log_path): + if Path(log_path).is_file(): fa2clus_log = read_json(log_path) else: fa2clus_log = {"algorithm": "null"} @@ -282,10 +286,7 @@ def run_fa2clus( return False, False, fa2clus_log except ClusteringError as le: # if cluster error, try aggclus if not error: - eprint( - spacer + "mmseqs failed, attempting hierarchical " + "clustering", - flush=True, - ) + logger.info(spacer + "mmseqs failed, attempting hierarchical " + "clustering") try: cluster, newick, overshot, fa2clus_log = fa2clus( fa_path, @@ -507,23 +508,23 @@ def make_output(base_dir, new_log): # eprint('\nERROR: base output directory missing: ' + base_dir, flush = True) # sys.exit(2) curdate = datetime.datetime.now().strftime("%Y%m%d") - output_dir = os.getcwd() + "/crap_" + curdate + "/" - if not os.path.isdir(output_dir): - os.mkdir(output_dir) + output_dir = str(Path.cwd()) + "/crap_" + curdate + "/" + if not Path(output_dir).is_dir(): + Path(output_dir).mkdir() else: - if not os.path.isdir(base_dir): - os.mkdir(base_dir) + if not Path(base_dir).is_dir(): + Path(base_dir).mkdir() output_dir = format_path(base_dir) # initialize a log in the output directory log_path = output_dir + ".craplog.json" parse_log(log_path, new_log, output_dir) - if not os.path.isdir(output_dir): - os.mkdir(output_dir) + if not Path(output_dir).is_dir(): + Path(output_dir).mkdir() loc_dir = output_dir + "loci/" - if not os.path.isdir(loc_dir): - os.mkdir(loc_dir) + if not Path(loc_dir).is_dir(): + Path(loc_dir).mkdir() wrk_dir = output_dir + "working/" global svg_dir # needs to be global for etetree @@ -532,12 +533,12 @@ def make_output(base_dir, new_log): svg_dir = wrk_dir + "svg/" gff_dir = wrk_dir + "genes/" tre_dir = wrk_dir + "trees/" - if not os.path.isdir(wrk_dir): - os.mkdir(wrk_dir) - if not os.path.isdir(svg_dir): - os.mkdir(svg_dir) - if not os.path.isdir(gff_dir): - os.mkdir(gff_dir) + if not Path(wrk_dir).is_dir(): + Path(wrk_dir).mkdir() + if not Path(svg_dir).is_dir(): + Path(svg_dir).mkdir() + if not Path(gff_dir).is_dir(): + Path(gff_dir).mkdir() return output_dir, wrk_dir, gff_dir, tre_dir @@ -593,20 +594,20 @@ def extract_locus_hg( try: gff_list = gff2list(gff3) except FileNotFoundError: - eprint(f"\t\t\tWARNING: {ome} MTDB entry without GFF3", flush=True) + logger.warning(f"{ome} MTDB entry without GFF3") return # grab the locus for the genes that do not currently have a locus SVG # (and thus also not a locus GFF) genes_to_grab = [ - x for x in genes_to_grab if not os.path.isfile(f"{wrk_dir}svg/{x}.locus.svg") + x for x in genes_to_grab if not Path(f"{wrk_dir}svg/{x}.locus.svg").is_file() ] try: out_indices, rna_gff = acc2locus( gff_list, genes_to_grab, plusminus, mycotools=True, geneGff=True, nt=True ) except ValueError: # KeyError: - eprint(f"\t\t\tWARNING: {ome} could not parse GFF", flush=True) + logger.warning(f"{ome} could not parse GFF") return # parse the loci and the genes associated with each locus, with the @@ -667,20 +668,20 @@ def extract_locus_gene( try: gff_list = gff2list(gff3) except FileNotFoundError: - eprint("\t\t\tWARNING: " + ome + " mycotoolsdb entry without GFF3", flush=True) + logger.warning("" + ome + " mycotoolsdb entry without GFF3") return - accs = [x for x in accs if not os.path.isfile(wrk_dir + "svg/" + x + ".locus.svg")] + accs = [x for x in accs if not Path(wrk_dir + "svg/" + x + ".locus.svg").is_file()] try: out_indices, rna_gff = acc2locus( gff_list, accs, plusminus, mycotools=True, geneGff=True, nt=True ) except KeyError: - eprint("\t\t\tWARNING: " + ome + " could not parse gff", flush=True) + logger.warning("" + ome + " could not parse gff") return extracted_genes, final_loci = {}, set() for locus_id, genes in out_indices.items(): - if os.path.isfile(wrk_dir + "svg/" + locus_id + ".locus.svg"): + if Path(wrk_dir + "svg/" + locus_id + ".locus.svg").is_file(): # NEED to rerun if there's a change in output parameters continue start_i, end_i = None, None @@ -757,7 +758,7 @@ def svgs2tree( circular=False, ): # svg_dir, out_dir): - init_dir = os.getcwd() + init_dir = str(Path.cwd()) os.chdir(out_dir) tree = Tree(tree_data) @@ -806,8 +807,8 @@ def svgs2tree( tree.render(out_dir + input_gene + "." + adj + ext, w=800, tree_style=ts) except TypeError: # QStandardPaths doesn't have permissions # this does not work - if not os.path.isdir(out_dir + ".XDG/"): - os.mkdir(out_dir + ".XDG/") + if not Path(out_dir + ".XDG/").is_dir(): + Path(out_dir + ".XDG/").mkdir() os.environ["XDG_RUNTIME_DIR"] = out_dir + ".XDG/" if og is not None: tree.render( @@ -837,9 +838,7 @@ def extend_color_palette(hgs, color_dict): try: color_dict[str(v)] = new_colors[i - cor_i] except IndexError: - eprint( - "\t\t\tWARNING: input too large for discrete arrow colors", flush=True - ) + logger.warning("input too large for discrete arrow colors") cor_i = i new_colors = colors color_dict[str(v)] = new_colors[i - cor_i] @@ -914,7 +913,7 @@ def make_color_palette(inputs, conversion_dict={}): for i, v in enumerate(inputs): color_dict[conversion_dict[v]] = extColors[i] else: - eprint("\nWARNING: input too large for discrete arrow colors", flush=True) + logger.warning("input too large for discrete arrow colors") try: for i, v in enumerate(inputs): color_dict[conversion_dict[v]] = extColors[i] @@ -938,7 +937,7 @@ def tree_mngr( ): query_fa_path = wrk_dir + query + ".fa" - if os.path.isfile(tre_dir + str(query) + tree_suffix) and reoutput: + if Path(tre_dir + str(query) + tree_suffix).is_file() and reoutput: return else: try: @@ -981,28 +980,28 @@ def parse_log(log_path, new_log, out_dir): # if db5 != old_log['db']: if md5 changes do somethign # rerun_search = True if old_log["search"] != new_log["search"]: - if os.path.isdir(out_dir): + if Path(out_dir).is_dir(): shutil.rmtree(out_dir) return elif old_log["bitscore"] != new_log["bitscore"]: fas = collect_files(wrk_dir, "fa") for fa in fas: - os.remove(fa) + Path(fa).unlink() fas = collect_files(clus_dir, "fa") for fa in fas: - os.remove(fa) - if os.path.isdir(wrk_dir + "tree/"): + Path(fa).unlink() + if Path(wrk_dir + "tree/").is_dir(): shutil.rmtree(wrk_dir + "tree/") elif old_log["plusminus"] != new_log["plusminus"]: - if os.path.isdir(wrk_dir + "genes/"): + if Path(wrk_dir + "genes/").is_dir(): shutil.rmtree(wrk_dir + "genes/") - if os.path.isdir(wrk_dir + "svg/"): + if Path(wrk_dir + "svg/").is_dir(): shutil.rmtree(wrk_dir + "svg/") elif old_log["labels"] != new_log["labels"]: - if os.path.isdir(wrk_dir + "svg/"): + if Path(wrk_dir + "svg/").is_dir(): shutil.rmtree(wrk_dir + "svg/") except KeyError: - eprint("\tERROR: log file corrupted. Hoping for the best.", flush=True) + logger.error("log file corrupted. Hoping for the best.") write_json(new_log, log_path) @@ -1039,7 +1038,7 @@ def crap_mngr( if info: return query2color - print("\t\tExtracting loci and generating synteny diagrams", flush=True) + logger.info("Extracting loci and generating synteny diagrams") extract_loci_cmds = [] if hg: for ome, ome_genes2hg in genes2query.items(): @@ -1091,7 +1090,7 @@ def crap_mngr( with mp.Pool(processes=cpus) as pool: gene_res = pool.starmap(extract_locus_gene, extract_loci_cmds) - print("\t\tMapping synteny diagrams on phylogeny", flush=True) + logger.info("Mapping synteny diagrams on phylogeny") tree_file = tre_dir + query + tree_suffix with open(tree_file, "r") as raw: raw_tree = raw.read() @@ -1117,7 +1116,7 @@ def crap_mngr( circular=circular, ) except NewickError: - eprint("\t\t\tERROR: newick malformatted", flush=True) + logger.error("newick malformatted") else: try: svgs2tree( @@ -1133,7 +1132,7 @@ def crap_mngr( circular=circular, # svg_dir, out_dir ) except NewickError: - eprint("\t\t\tERROR: newick malformatted", flush=True) + logger.error("newick malformatted") return query2color @@ -1146,7 +1145,7 @@ def write_loci(ome2genes, loc_dir): queries = ",".join(query) out.write(f"{gene}\t{queries}\n") else: - eprint(f"\t{ome} no hits", flush=True) + logger.info(f"{ome} no hits") # out.write('\n'.join(genes)) @@ -1206,7 +1205,7 @@ def parse_search_col_loci( id_mean = sum(identities) / len(identities) loc2sim[f] = [alia, locus_sim * id_mean] except FileNotFoundError: - eprint(f"\tWARNING: {ome} locus not weighted by %ID", flush=True) + logger.warning(f"{ome} locus not weighted by %ID") loc2sim[f] = [alia, locus_sim] # sort the loci by similarity @@ -1279,7 +1278,7 @@ def locus_output_mngr( ome2files = defaultdict(list) ome2locs = defaultdict(list) for f in files: - ome = os.path.basename(f)[: os.path.basename(f).find("_")] + ome = Path(f).name[: Path(f).name.find("_")] ome2files[ome].append(f) if not report_dir: @@ -1349,8 +1348,8 @@ def hg_main( db = db.set_index() - print("\nCompiling homolog data", flush=True) - print("\tCompiling homologs", flush=True) + logger.info("Compiling homolog data") + logger.info("Compiling homologs") # og_info_dict = mycodbHGs(omes = set(db['ome'])) # hg2gene, gene2hg = extract_ogs(og_info_dict, ogtag) ome2i, gene2hg, i2ome, hg2gene = compile_homolog_groups( @@ -1375,12 +1374,12 @@ def hg_main( # in the future, genes without HGs will be placed into HGs via RBH if not input_hgs: - eprint("\nERROR: no HGs for any inputted genes", flush=True) + logger.error("no HGs for any inputted genes") sys.exit(3) if output_loci: - if all(os.path.isfile(f"{tre_dir}{gene}{tree_suffix}") for gene in input_hgs): - print("\nSkipping to outputting most similar loci to query", flush=True) + if all(Path(f"{tre_dir}{gene}{tree_suffix}").is_file() for gene in input_hgs): + logger.info("Skipping to outputting most similar loci to query") locus_output_mngr( gff_dir, loc_dir, @@ -1393,60 +1392,56 @@ def hg_main( return hg_fas = {} - if not all(os.path.isfile(f"{faa_dir}{hg}.faa") for gene, hg in input_hgs.items()): - print("\tPreparing homolog fastas", flush=True) + if not all(Path(f"{faa_dir}{hg}.faa").is_file() for gene, hg in input_hgs.items()): + logger.info("Preparing homolog fastas") compile_hg_fa_cmds = [ [db, hg2gene[hg], gene] for gene, hg in input_hgs.items() - if not os.path.isfile(wrk_dir + gene + ".fa") + if not Path(wrk_dir + gene + ".fa").is_file() ] with mp.Pool(processes=cpus) as pool: hg_fas = { x[0]: x[1] for x in pool.starmap(compile_hg_fa, compile_hg_fa_cmds) } for gene, hg in input_hgs.items(): - if os.path.isfile( - wrk_dir + gene + ".fa" - ): # add finished in working directory back + if Path(wrk_dir + gene + ".fa").is_file(): # add finished in working directory back hg_fas = {**hg_fas, **{gene: fa2dict(wrk_dir + gene + ".fa")}} else: for gene, hg in input_hgs.items(): hg_fas = {**hg_fas, **{gene: fa2dict(f"{faa_dir}{hg}.faa")}} for gene in input_hgs: if gene not in hg_fas[gene]: - eprint( - f"\t\tERROR: {gene} not in homologs. Incorrect input?", flush=True - ) + logger.error(f"{gene} not in homologs. Incorrect input?") sys.exit(3) - print("\nChecking fasta sizes", flush=True) + logger.info("Checking fasta sizes") fas4clus, fas4trees = check_fa_size(hg_fas, max_size) for query, hit_fa in fas4trees.items(): hit_fa_path = wrk_dir + query + ".fa" with open(hit_fa_path, "w") as out: out.write(dict2fa(hit_fa)) - if not os.path.isdir(clus_dir): - os.mkdir(clus_dir) + if not Path(clus_dir).is_dir(): + Path(clus_dir).mkdir() if fas4clus: for query, fa in fas4clus.items(): clus_fa_path = clus_dir + query + ".fa" - if not os.path.isfile(clus_fa_path): + if not Path(clus_fa_path).is_file(): with open(clus_fa_path, "w") as out: out.write(dict2fa(fa)) - print("\nRunning clustering on " + str(len(fas4clus)) + " fastas", flush=True) + logger.debug("Running clustering on " + str(len(fas4clus)) + " fastas") - print("\nCRAP", flush=True) + logger.info("CRAP") ome_gene2hg = gene2hg2ome2hg(gene2hg) fas4trees = {k: v for k, v in sorted(fas4trees.items(), key=lambda x: len(x[1]))} for query, query_fa in fas4trees.items(): out_keys = None query_hits = list(query_fa.keys()) - print("\tQuery: " + str(query), flush=True) + logger.debug("Query: " + str(query)) if outgroups: - print("\t\tOutgroup detection", flush=True) - if os.path.isfile(clus_dir + query + ".fa"): - if not os.path.isfile(wrk_dir + query + ".outgroup.fa"): + logger.info("Outgroup detection") + if Path(clus_dir + query + ".fa").is_file(): + if not Path(wrk_dir + query + ".outgroup.fa").is_file(): in_keys, all_keys = outgroup_mngr( db, query, @@ -1462,11 +1457,11 @@ def hg_main( out_query = query + ".outgroup" query_hits = all_keys out_keys = list(set(all_keys).difference(set(in_keys))) - print("\t\t\t" + str(len(in_keys)) + " gene ingroup", flush=True) + logger.debug("" + str(len(in_keys)) + " gene ingroup") if out_keys: - print("\t\t\t" + str(len(out_keys)) + " gene outgroup", flush=True) + logger.debug("" + str(len(out_keys)) + " gene outgroup") else: - eprint("\t\t\tWARNING: Could not detect outgroup for root", flush=True) + logger.warning("Could not detect outgroup for root") HG = input_hgs[ re.sub(r"\.outgroup$", "", query) ] # bulletproof against outgroups @@ -1496,8 +1491,8 @@ def hg_main( ) for query in fas4clus: - print("\tQuery: " + str(query), flush=True) - print("\t\tSequence clustering", flush=True) + logger.debug("Query: " + str(query)) + logger.info("Sequence clustering") out_keys = None res, overshot, fa2clus_log = run_fa2clus( clus_dir + query + ".fa", @@ -1515,11 +1510,11 @@ def hg_main( algorithm=clus_meth, ) if not res: - print("\t\t\tERROR: query had no significant hits", flush=True) + logger.error("query had no significant hits") continue if outgroups and not overshot: - print("\t\tOutgroup detection", flush=True) - if not os.path.isfile(wrk_dir + query + ".outgroup.fa"): + logger.info("Outgroup detection") + if not Path(wrk_dir + query + ".outgroup.fa").is_file(): in_keys, all_keys = outgroup_mngr( db, query, @@ -1535,9 +1530,9 @@ def hg_main( out_query = query + ".outgroup" query_hits = all_keys out_keys = list(set(all_keys).difference(in_keys)) - print("\t\t\t" + str(len(in_keys)) + " gene ingroup", flush=True) + logger.debug("" + str(len(in_keys)) + " gene ingroup") if out_keys: - print("\t\t\t" + str(len(out_keys)) + " gene outgroup", flush=True) + logger.debug("" + str(len(out_keys)) + " gene outgroup") else: query_fa = fa2dict(wrk_dir + query + ".fa") query_hits = list(query_fa.keys()) @@ -1570,7 +1565,7 @@ def hg_main( ) if output_loci: - print("\nOutputting most similar loci to query", flush=True) + logger.info("Outputting most similar loci to query") locus_output_mngr( gff_dir, loc_dir, @@ -1616,7 +1611,7 @@ def search_main( ): """input_genes is a list of genes within an inputted cluster""" - print("\nPreparing run", flush=True) + logger.info("Preparing run") wrk_dir, loc_dir = out_dir + "working/", out_dir + "loci/" gff_dir, tre_dir = wrk_dir + "genes/", wrk_dir + "trees/" clus_dir = wrk_dir + "clus/" @@ -1628,7 +1623,7 @@ def search_main( query2color, conversion_dict = make_color_palette(input_genes, conversion_dict) if query_gff: - print("\tCleaning input GFF", flush=True) + logger.info("Cleaning input GFF") par_dict, prot_hits, RNA, query_gff = prep_gff( query_gff, set(input_genes), gff3Comps() ) @@ -1639,16 +1634,14 @@ def search_main( query_gff, set(input_genes), gff3Comps(), prot_hits, par_dict ) if par_dict: - eprint("\nIncorrectly formatted GFF", flush=True) + logger.info("Incorrectly formatted GFF") sys.exit(5) elif set(input_genes).difference(prot_hits): - eprint("\nProteins missing from GFF", flush=True) - eprint( - "\t" + logger.info("Proteins missing from GFF") + logger.info("" + ",".join( [str(x) for x in list(set(input_genes).difference(prot_hits))] - ) - ) + )) sys.exit(6) clean_gff = [ x @@ -1663,22 +1656,22 @@ def search_main( query_fa = acc2fa(db, input_genes) with open(query_path, "w") as out: out.write(dict2fa(query_fa)) - elif not os.path.isfile(query_path): + elif not Path(query_path).is_file(): with open(query_path, "w") as out: out.write(dict2fa(query_fa)) search_fas = {} for query in query_fa: - if os.path.isfile(clus_dir + query + ".fa"): + if Path(clus_dir + query + ".fa").is_file(): search_fas[query] = fa2dict(clus_dir + query + ".fa") search_fas[query][query] = query_fa[query] - elif os.path.isfile(wrk_dir + query + ".fa"): + elif Path(wrk_dir + query + ".fa").is_file(): search_fas[query] = fa2dict(wrk_dir + query + ".fa") search_fas[query][query] = query_fa[query] if output_loci: - if all(os.path.isfile(f"{tre_dir}{gene}{tree_suffix}") for gene in query_fa): - print("\nSkipping to outputting most similar loci to query", flush=True) + if all(Path(f"{tre_dir}{gene}{tree_suffix}").is_file() for gene in query_fa): + logger.info("Skipping to outputting most similar loci to query") locus_output_mngr( gff_dir, loc_dir, @@ -1720,14 +1713,14 @@ def search_main( for query in search_fas: search_fas[query][query] = query_fa[query] - print("\nChecking hit fasta sizes", flush=True) + logger.info("Checking hit fasta sizes") genes2query, merges = compile_genes_by_omes( search_fas, conversion_dict, set(db["ome"]) ) query2color = merge_color_palette(merges, query2color) for query in search_fas: # revert back to other fas - if os.path.isfile(wrk_dir + query + ".fa"): + if Path(wrk_dir + query + ".fa").is_file(): search_fas[query] = fa2dict(wrk_dir + query + ".fa") fas4clus, fas4trees = check_fa_size(search_fas, max_size) for query, hit_fa in fas4trees.items(): @@ -1736,26 +1729,26 @@ def search_main( out.write(dict2fa(hit_fa)) if fas4clus: - if not os.path.isdir(clus_dir): - os.mkdir(clus_dir) + if not Path(clus_dir).is_dir(): + Path(clus_dir).mkdir() for query, fa in fas4clus.items(): clus_fa_path = clus_dir + query + ".fa" - if not os.path.isfile(clus_fa_path): + if not Path(clus_fa_path).is_file(): with open(clus_fa_path, "w") as out: out.write(dict2fa(fa)) - print("\tRunning clustering on " + str(len(fas4clus)) + " fastas", flush=True) + logger.debug("Running clustering on " + str(len(fas4clus)) + " fastas") - print("\nCRAP", flush=True) + logger.info("CRAP") ome2genes = {} fas4trees = {k: v for k, v in sorted(fas4trees.items(), key=lambda x: len(x[1]))} for query, query_fa in fas4trees.items(): out_keys = None query_hits = list(query_fa.keys()) - print("\tQuery: " + str(query), flush=True) + logger.debug("Query: " + str(query)) if outgroups: - print("\t\tOutgroup detection", flush=True) - if os.path.isfile(clus_dir + query + ".fa"): - if not os.path.isfile(wrk_dir + query + ".outgroup.fa"): + logger.info("Outgroup detection") + if Path(clus_dir + query + ".fa").is_file(): + if not Path(wrk_dir + query + ".outgroup.fa").is_file(): in_keys, all_keys = outgroup_mngr( db, query, @@ -1771,11 +1764,11 @@ def search_main( out_query = query + ".outgroup" query_hits = all_keys out_keys = list(set(all_keys).difference(set(in_keys))) - print("\t\t\t" + str(len(in_keys)) + " gene ingroup", flush=True) + logger.debug("" + str(len(in_keys)) + " gene ingroup") if out_keys: - print("\t\t\t" + str(len(out_keys)) + " gene outgroup", flush=True) + logger.debug("" + str(len(out_keys)) + " gene outgroup") else: - eprint("\t\t\tWARNING: could not detect outgroup for root", flush=True) + logger.warning("could not detect outgroup for root") null = crap_mngr( db, query, @@ -1806,8 +1799,8 @@ def search_main( clus_cpus = cpus for query in fas4clus: - print("\tQuery: " + str(query), flush=True) - print("\t\tSequence clustering", flush=True) + logger.debug("Query: " + str(query)) + logger.info("Sequence clustering") out_keys = None query_hits = list(query_fa.keys()) @@ -1827,12 +1820,12 @@ def search_main( algorithm=clus_meth, ) if not res: - print("\t\t\tFAILED: query had no significant hits", flush=True) + logger.warning("FAILED: query had no significant hits") continue if outgroups and not overshot: - print("\t\tOutgroup detection", flush=True) - if not os.path.isfile(wrk_dir + query + ".outgroup.fa"): + logger.info("Outgroup detection") + if not Path(wrk_dir + query + ".outgroup.fa").is_file(): # WILL RAISE AN ERROR IF THIS EXISTS BECAUSE ALL_KEYS DOESNT in_keys, all_keys = outgroup_mngr( db, @@ -1849,9 +1842,9 @@ def search_main( query_hits = all_keys out_query = query + ".outgroup" out_keys = list(set(all_keys).difference(set(in_keys))) - print("\t\t\t" + str(len(in_keys)) + " gene ingroup", flush=True) + logger.debug("" + str(len(in_keys)) + " gene ingroup") if out_keys: - print("\t\t\t" + str(len(out_keys)) + " gene outgroup", flush=True) + logger.debug("" + str(len(out_keys)) + " gene outgroup") else: query_fa = fa2dict(wrk_dir + query + ".fa") null = crap_mngr( @@ -1878,7 +1871,7 @@ def search_main( ) if output_loci: - print("\nOutputting most similar loci to query", flush=True) + logger.info("Outputting most similar loci to query") locus_output_mngr( gff_dir, loc_dir, @@ -2044,11 +2037,12 @@ def cli(): help='Output format: ["svg", "pdf", "png"]', ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) execs = ["diamond", "clipkit", "mafft", "iqtree"] if args.search: if args.search not in {"mmseqs", "blastp", "diamond"}: - eprint("\nERROR: invalid -s", flush=True) + logger.error("invalid -s") sys.exit(3) else: execs.append(args.search) @@ -2056,25 +2050,25 @@ def cli(): findExecs(execs, exit=set(execs)) if args.out_format.lower() not in {"svg", "pdf", "png"}: - eprint("\nERROR: invalid -of", flush=True) + logger.error("invalid -of") sys.exit(10) else: out_ext = "." + args.out_format.lower() if not args.homologs and args.faa: - eprint("\nERROR: -hf requires -hg", flush=True) + logger.error("-hf requires -hg") sys.exit(12) input_fa, input_GFF = False, False if args.query == "-": if args.gff: - eprint("\nERROR: GFF input requires fasta input", flush=True) + logger.error("GFF input requires fasta input") sys.exit(3) input_genes = stdin2str().replace('"', "").replace("'", "").split() elif "'" in args.query or '"' in args.query: if args.gff: - eprint("\nERROR: GFF input requires fasta input", flush=True) + logger.error("GFF input requires fasta input") sys.exit(3) input_genes = ( args.query.replace('"', "").replace("'", "").replace(",", " ").split() @@ -2088,7 +2082,7 @@ def cli(): gene0 = input_genes[0] ome = input_genes[0][: input_genes[0].find("_")] if not ome in set(db["ome"]): - if os.path.isfile(args.query): + if Path(args.query).is_file(): if args.query.lower().endswith((".fasta", ".fa", ".faa", ".fna", ".fsa")): input_fa = fa2dict(args.query) input_genes = list(input_fa.keys()) @@ -2099,12 +2093,9 @@ def cli(): data = raw.read() input_genes = data.rstrip().split() else: - print("\nDetected non-mycotools input", flush=True) + logger.info("Detected non-mycotools input") if not input_fa or not args.search: - eprint( - "\tnon-mycotools input requires -s, -i as a fasta, optionally -g", - flush=True, - ) + logger.info("non-mycotools input requires -s, -i as a fasta, optionally -g") sys.exit(4) # eprint('\nERROR: invalid input', flush = True) @@ -2123,7 +2114,7 @@ def cli(): fast = True if args.agg_clus and args.linclust: - eprint("\nERROR: multiple clustering methods specified", flush=True) + logger.error("multiple clustering methods specified") sys.exit(5) elif args.agg_clus: clus_meth = "diamond" @@ -2174,7 +2165,7 @@ def cli(): args.plusminus, not args.no_label, ) - print("\nPreparing output directory", flush=True) + logger.info("Preparing output directory") out_dir, wrk_dir, gff_dir, tre_dir = make_output(output, new_log) hg_main( db, @@ -2210,7 +2201,7 @@ def cli(): args.plusminus, not args.no_label, ) - print("\nPreparing output directory", flush=True) + logger.info("Preparing output directory") out_dir, wrk_dir, gff_dir, tre_dir = make_output(output, new_log) search_main( db, diff --git a/mycotools/db2files.py b/mycotools/db2files.py index 89e9c9d..a4ed619 100755 --- a/mycotools/db2files.py +++ b/mycotools/db2files.py @@ -1,5 +1,6 @@ #! /usr/bin/env python3 +import logging import os import re import sys @@ -7,7 +8,10 @@ from datetime import datetime from shutil import copy as cp from mycotools.lib.dbtools import primaryDB, mtdb -from mycotools.lib.kontools import format_path, prep_output, eprint, vprint +from mycotools.lib.kontools import format_path, prep_output, setup_logging +from pathlib import Path + +logger = logging.getLogger(__name__) def soft_main(filetypes, db, output_path, print_link=False, verbose=False): @@ -18,27 +22,23 @@ def soft_main(filetypes, db, output_path, print_link=False, verbose=False): if not print_link: # make the directories for each requested file type for ftype in filetypes: - if not os.path.isdir(output_path + ftype): - os.mkdir(output_path + ftype) + if not Path(output_path + ftype).is_dir(): + Path(output_path + ftype).mkdir() # grab the files for each genome code for ome, row in db.items(): for ftype in filetypes: - if os.path.isfile(row[ftype]): + if Path(row[ftype]).is_file(): sym_path = f"{output_path}{ftype}/{ome}.{ftype}" try: - os.symlink(row[ftype], sym_path) + Path(sym_path).symlink_to(row[ftype]) except FileExistsError: - if os.path.islink(sym_path): - os.remove(sym_path) - os.symlink(row[ftype], sym_path) + if Path(sym_path).is_symlink(): + Path(sym_path).unlink() + Path(sym_path).symlink_to(row[ftype]) else: - vprint( - "\t" + ome + " " + ftype + " exists", - v=verbose, - flush=True, - ) + logger.debug("" + ome + " " + ftype + " exists") else: - vprint("\tERROR: " + ome + " " + ftype, flush=True, v=verbose) + logger.debug("" + ome + " " + ftype) # simply print the link for each file else: for ome, row in db.items(): @@ -52,16 +52,16 @@ def hard_main(filetypes, db, output_path): db = db.set_index("ome") # create the directories to output each file type for ftype in filetypes: - if not os.path.isdir(output_path + ftype): - os.mkdir(output_path + ftype) + if not Path(output_path + ftype).is_dir(): + Path(output_path + ftype).mkdir() # copy each file by genome for ome, row in db.items(): for ftype in filetypes: try: - cp(row[ftype], output_path + ftype + "/" + os.path.basename(row[ftype])) + cp(row[ftype], output_path + ftype + "/" + Path(row[ftype]).name) except FileNotFoundError: - eprint("\tERROR: " + ome + " " + ftype, flush=True) + logger.error("" + ome + " " + ftype) def mtdb_main(db, output_path, og_mtdb_path): @@ -69,12 +69,12 @@ def mtdb_main(db, output_path, og_mtdb_path): # generate the base directory for output if not output_path: - output_path = os.getcwd() + "/" - if not os.path.isdir(output_path): - os.mkdir(output_path) + output_path = str(Path.cwd()) + "/" + if not Path(output_path).is_dir(): + Path(output_path).mkdir() mtdb_dir = output_path + "mycotoolsdb/" - if not os.path.isdir(mtdb_dir): - os.mkdir(mtdb_dir) + if not Path(mtdb_dir).is_dir(): + Path(mtdb_dir).mkdir() # generate the MTDB hierarchy subdirectories sub_dirs = [ @@ -84,8 +84,8 @@ def mtdb_main(db, output_path, og_mtdb_path): f"{mtdb_dir}data/", ] for dir_ in sub_dirs: - if not os.path.isdir(dir_): - os.mkdir(dir_) + if not Path(dir_).is_dir(): + Path(dir_).mkdir() # copy the og_mtdb configuration cp(og_mtdb_path + "config/mtdb.json", f"{mtdb_dir}config/mtdb.json") @@ -112,11 +112,12 @@ def cli(): parser.add_argument( "-n", "--new_mtdb", action="store_true", help="Create MTDB directory hierarchy" ) - parser.add_argument("-o", "--output", default=os.getcwd()) + parser.add_argument("-o", "--output", default=str(Path.cwd())) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if not args.assembly and not args.proteome and not args.gff and not args.new_mtdb: - print("\nERROR: --assembly/--proteome/--gff/--new_mtdb required", flush=True) + logger.error("--assembly/--proteome/--gff/--new_mtdb required") sys.exit(4) if args.new_mtdb: args.hard = False diff --git a/mycotools/db2hgs.py b/mycotools/db2hgs.py index e5f9eb5..aa1e7ed 100755 --- a/mycotools/db2hgs.py +++ b/mycotools/db2hgs.py @@ -3,6 +3,7 @@ import os import sys import shutil +import logging import argparse import subprocess import multiprocessing as mp @@ -12,7 +13,11 @@ from mycotools.db2files import soft_main as symlink_files from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.biotools import fa2dict, dict2fa, fa2dict_accs -from mycotools.lib.kontools import format_path, eprint, mkOutput, findExecs +from mycotools.lib.kontools import format_path, mkOutput, findExecs, setup_logging +from pathlib import Path + + +logger = logging.getLogger(__name__) def mk_db2hg_output(out_dir, nscg=False): @@ -22,14 +27,14 @@ def mk_db2hg_output(out_dir, nscg=False): nscg_dir = out_dir + "near_single_copy_genes/" hg_seq_dir = out_dir + "hgs/" - if not os.path.isdir(wrk_dir): - os.mkdir(wrk_dir) - if not os.path.isdir(scg_dir): - os.mkdir(scg_dir) - if not os.path.isdir(nscg_dir) and nscg: - os.mkdir(nscg_dir) - if not os.path.isdir(hg_seq_dir): - os.mkdir(hg_seq_dir) + if not Path(wrk_dir).is_dir(): + Path(wrk_dir).mkdir() + if not Path(scg_dir).is_dir(): + Path(scg_dir).mkdir() + if not Path(nscg_dir).is_dir() and nscg: + Path(nscg_dir).mkdir() + if not Path(hg_seq_dir).is_dir(): + Path(hg_seq_dir).mkdir() return wrk_dir, scg_dir, nscg_dir, hg_seq_dir @@ -46,12 +51,12 @@ def run_mmseqs( """Run MMseqs clustering by sym linking MTDB proteomes""" symlink_files(["faa"], db, wrk_dir, verbose=False) # symlink proteomes cluster_res_file = wrk_dir + "raw_hgs.tsv" - if not os.path.isfile(cluster_res_file): # NEED to add to log removal + if not Path(cluster_res_file).is_file(): # NEED to add to log removal # be cautious about shell injection because we need to glob int(cpus) float(min_id) float(min_cov) - if not os.path.isdir(wrk_dir): + if not Path(wrk_dir).is_dir(): raise OSError("invalid working directory") elif not algorithm in {"mmseqs easy-linclust", "mmseqs easy-cluster"}: raise OSError("invalid mmseqs binary") @@ -82,18 +87,18 @@ def run_mmseqs( ) # stderr = subprocess.DEVNULL) shutil.move(wrk_dir + "cluster_cluster.tsv", cluster_res_file) - elif os.path.getsize(cluster_res_file): + elif Path(cluster_res_file).stat().st_size: mmseqs_cmd = 0 else: mmseqs_cmd = 1 if mmseqs_cmd: - eprint("\tERROR: cluster failed") + logger.error("cluster failed") sys.exit(1) - if os.path.isfile(wrk_dir + "cluster_all_seqs.fasta"): - os.remove(wrk_dir + "cluster_all_seqs.fasta") - if os.path.isfile(wrk_dir + "cluster_rep_seq.fasta"): - os.remove(wrk_dir + "cluster_rep_seq.fasta") - if os.path.isdir(wrk_dir + "tmp/"): + if Path(wrk_dir + "cluster_all_seqs.fasta").is_file(): + Path(wrk_dir + "cluster_all_seqs.fasta").unlink() + if Path(wrk_dir + "cluster_rep_seq.fasta").is_file(): + Path(wrk_dir + "cluster_rep_seq.fasta").unlink() + if Path(wrk_dir + "tmp/").is_dir(): shutil.rmtree(wrk_dir + "tmp/") return cluster_res_file @@ -239,7 +244,7 @@ def write_hgs(hg, genes, wrk_dir, write_dir): with open(f"{write_dir}{hg}.faa.tmp", "w") as out: out.write(dict2fa(fa_dict)) - os.rename(f"{write_dir}{hg}.faa.tmp", f"{write_dir}{hg}.faa") + Path(f"{write_dir}{hg}.faa.tmp").rename(f"{write_dir}{hg}.faa") def align_hg(hg_fa, out_fa, cpus=1): @@ -248,7 +253,7 @@ def align_hg(hg_fa, out_fa, cpus=1): cmd = subprocess.call( ["mafft", "--auto", "--thread", f"-{cpus}", hg_fa], stdout=out ) # , stderr = subprocess.PIPE) - os.rename(out_fa + ".tmp", out_fa) + Path(out_fa + ".tmp").rename(out_fa) return cmd @@ -259,7 +264,7 @@ def hmmbuild_hg(msa_fa, out_hmm, cpus=1): stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - os.rename(out_hmm + ".tmp", out_hmm) + Path(out_hmm + ".tmp").rename(out_hmm) return cmd @@ -350,16 +355,16 @@ def main( aln_file = out_dir + "accessory_alignment.phy" wrk_dir, scg_dir, nscg_dir, hg_seq_dir = mk_db2hg_output(out_dir, nscg) - print("\nClustering protein sequences", flush=True) + logger.info("Clustering protein sequences") raw_hg_file = run_mmseqs(db, wrk_dir, algorithm, min_id, min_cov, sensitivity, cpus) useable_omes = set(db.keys()) - print("\nCompiling homology groups (HGs)", flush=True) + logger.info("Compiling homology groups (HGs)") ome_num, gene2hg, i2ome, hg2gene = compile_homolog_groups( raw_hg_file, hg_output, useable_omes ) - print("\nIdentifying single-copy HGs", flush=True) + logger.info("Identifying single-copy HGs") schgs, nschgs, hg2stats, full_hgs, hg2d_omes = id_near_schgs( hg2gene, set(i2ome), @@ -370,7 +375,7 @@ def main( min_genomes=min_genomes, ) - print("\nWriting output", flush=True) + logger.info("Writing output") ome2pan = pangenome_output(pan_file, aln_file, hg2gene, hg2d_omes, max_mis_ome=0) with open(hg2missing_genome_file, "w") as out: @@ -385,28 +390,28 @@ def main( {k: hg2stats[k] for k in list(full_hgs)}, full_hg_stats_file, sort=True ) if nscg: - print(f"\t{len(nschgs)} near single-copy HGs", flush=True) + logger.info(f"{len(nschgs)} near single-copy HGs") with mp.Pool(processes=cpus) as pool: pool.starmap( write_hgs, ( (hg, hg2gene[hg], wrk_dir, nscg_dir) for hg in nschgs - if not os.path.isfile(f"{nscg_dir}{hg}.faa") + if not Path(f"{nscg_dir}{hg}.faa").is_file() ), ) # for hg in nschgs: # if not os.path.isfile(f'{nscg_dir}{hg}.faa'): # write_hgs(db, hg, hg2gene[hg], wrk_dir, nscg_dir) if schgs: - print(f"\t{len(schgs)} single-copy HGs", flush=True) + logger.info(f"{len(schgs)} single-copy HGs") with mp.Pool(processes=cpus) as pool: pool.starmap( write_hgs, ( (hg, hg2gene[hg], wrk_dir, scg_dir) for hg in schgs - if not os.path.isfile(f"{scg_dir}{hg}.faa") + if not Path(f"{scg_dir}{hg}.faa").is_file() ), ) # for hg in schgs: @@ -419,7 +424,7 @@ def main( ( (hg, genes, wrk_dir, hg_seq_dir) for hg, genes in hg2genes.items() - if not os.path.isfile(f"{hg_seq_dir}{hg}.faa") + if not Path(f"{hg_seq_dir}{hg}.faa").is_file() ), ) @@ -428,12 +433,12 @@ def main( # write_hgs(db, hg, genes, wrk_dir, hg_seq_dir) if hmm: - print("\nAligning and building HMMs", flush=True) + logger.info("Aligning and building HMMs") msa_dir = out_dir + "msa/" hmm_dir = out_dir + "hmm/" for d in [msa_dir, hmm_dir]: - if not os.path.isdir(d): - os.mkdir(d) + if not Path(d).is_dir(): + Path(d).mkdir() if nscg: srch_hgs = nschgs hg_dir = nscg_dir @@ -441,11 +446,11 @@ def main( srch_hgs = schgs hg_dir = scg_dir for hg in srch_hgs: - if not os.path.isfile(f"{msa_dir}{hg}.mafft.faa"): + if not Path(f"{msa_dir}{hg}.mafft.faa").is_file(): mafft_code = align_hg( f"{nscg_dir}{hg}.faa", f"{msa_dir}{hg}.mafft.faa", cpus=cpus ) - if not os.path.isfile(f"{hmm_dir}{hg}.hmm"): + if not Path(f"{hmm_dir}{hg}.hmm").is_file(): hmm_code = hmmbuild_hg( f"{msa_dir}{hg}.mafft.faa", f"{hmm_dir}{hg}.hmm", cpus=cpus ) @@ -492,17 +497,18 @@ def cli(): parser.add_argument("-o", "--out_dir", help="WARNING: will not overwrite") parser.add_argument("-c", "--cpus", type=int, default=mp.cpu_count()) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.min_genomes > 1 or args.min_genomes < 0: - eprint("\nERROR: --min_genomes must be between 0 and 1", flush=True) + logger.error("--min_genomes must be between 0 and 1") sys.exit(143) db = mtdb(format_path(args.mtdb)) if args.out_dir: out_dir = format_path(args.out_dir) - if not os.path.isdir(out_dir): - os.mkdir(out_dir) + if not Path(out_dir).is_dir(): + Path(out_dir).mkdir() out_dir += "/" else: out_dir = mkOutput(args.out_dir, "db2hgs") diff --git a/mycotools/db2microsyntree.py b/mycotools/db2microsyntree.py index 0142fda..2da3c9b 100755 --- a/mycotools/db2microsyntree.py +++ b/mycotools/db2microsyntree.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import logging import os import sys import shutil @@ -18,10 +19,13 @@ findExecs, intro, outro, - eprint, + setup_logging, ) from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.biotools import gff2list +from pathlib import Path + +logger = logging.getLogger(__name__) def run_mmseqs( @@ -35,12 +39,12 @@ def run_mmseqs( ): symlink_files(["faa"], db, wrk_dir, verbose=False) # symlink proteomes cluster_res_file = wrk_dir + "homolog_groups.tsv" - if not os.path.isfile(cluster_res_file): # NEED to add to log removal + if not Path(cluster_res_file).is_file(): # NEED to add to log removal # be cautious about shell injection because we need to glob int(cpus) float(min_id) float(min_cov) - if not os.path.isdir(wrk_dir): + if not Path(wrk_dir).is_dir(): raise OSError("invalid working directory") elif not algorithm in {"mmseqs easy-linclust", "mmseqs easy-cluster"}: raise OSError("invalid mmseqs binary") @@ -71,18 +75,18 @@ def run_mmseqs( ) # stderr = subprocess.DEVNULL) shutil.move(wrk_dir + "cluster_cluster.tsv", cluster_res_file) - elif os.path.getsize(cluster_res_file): + elif Path(cluster_res_file).stat().st_size: mmseqs_cmd = 0 else: mmseqs_cmd = 1 if mmseqs_cmd: - eprint("\tERROR: cluster failed") + logger.error("cluster failed") sys.exit(1) - if os.path.isfile(wrk_dir + "cluster_all_seqs.fasta"): - os.remove(wrk_dir + "cluster_all_seqs.fasta") - if os.path.isfile(wrk_dir + "cluster_rep_seq.fasta"): - os.remove(wrk_dir + "cluster_rep_seq.fasta") - if os.path.isdir(wrk_dir + "tmp/"): + if Path(wrk_dir + "cluster_all_seqs.fasta").is_file(): + Path(wrk_dir + "cluster_all_seqs.fasta").unlink() + if Path(wrk_dir + "cluster_rep_seq.fasta").is_file(): + Path(wrk_dir + "cluster_rep_seq.fasta").unlink() + if Path(wrk_dir + "tmp/").is_dir(): shutil.rmtree(wrk_dir + "tmp/") return cluster_res_file @@ -198,12 +202,9 @@ def compile_cds(gff_list, ome, gene2hg): elif not prot: # if there isn't a valid accession it may mean the # mycotools curation did not work or the user did not curate # correctly - print(entry["attributes"], prot_prep_i0, prot_prep_i1) + logger.debug("%s %s %s", entry["attributes"], prot_prep_i0, prot_prep_i1) if not fail: - print( - "\tWARNING: " + ome + " has proteins in gff with no Alias", - flush=True, - ) + logger.debug("" + ome + " has proteins in gff with no Alias") fail = True continue cds_dict[entry["seqid"]][prot].extend( @@ -234,7 +235,7 @@ def parse_loci(gff_path, ome, gene2hg, window=6): gff_list = gff2list(gff_path) # open here to improve pickling hg_dict = compile_cds( - gff_list, os.path.basename(gff_path).replace(".gff3", ""), gene2hg + gff_list, Path(gff_path).name.replace(".gff3", ""), gene2hg ) pairs = [] for scaf, hgs in hg_dict.items(): # for each contig @@ -342,26 +343,20 @@ def align_microsynt_np(m_arr, i2ome, hg2gene, hgpair2i, wrk_dir, nschgs=None): ) if len(schgs) > 9: nschgs = schgs - print(f"\t\t{len(schgs)} HGs in single copy extracted", flush=True) + logger.debug(f"{len(schgs)} HGs in single copy extracted") elif len(nschgs) < 10: nschgs = [] if nschgs: - print( - f"\t\t{len(nschgs)} HGs with <= {max_median} median copy " - + f"number and <= {max_stdev} standard deviation extracted", - flush=True, - ) + logger.debug(f"{len(nschgs)} HGs with <= {max_median} median copy " + + f"number and <= {max_stdev} standard deviation extracted") max_stdev += 0.2 if max_stdev > 2: max_stdev = 0.1 max_median += 1 if not nschgs: - eprint( - "\nERROR: could not detect 10 genes present in all genomes " + logger.error("could not detect 10 genes present in all genomes " + "with median 2 copy number and less than 2 copy number " - + "standard deviation. Manually input focal homology groups.", - flush=True, - ) + + "standard deviation. Manually input focal homology groups.") sys.exit(35) pre_arr = extract_nschg_pairs(nschgs, hgpair2i, m_arr) @@ -395,8 +390,8 @@ def run_tree( ): tree_dir = wrk_dir + "tree/" - if not os.path.isdir(tree_dir): - os.mkdir(tree_dir) + if not Path(tree_dir).is_dir(): + Path(tree_dir).mkdir() prefix = tree_dir + "microsynt" tree_cmd = [iqtree, "-s", alignment, "-m", model, "--prefix", prefix] @@ -437,10 +432,10 @@ def main( # obtain useable omes useableOmes, dbOmes = set(), set(db.keys()) - print("\nI. Inputting data", flush=True) + logger.info("I. Inputting data") if n50thresh: # optional n50 threshold via mycotoolsDB assemblyPath = format_path("$MYCODB/../data/assemblyStats.tsv") - if os.path.isfile(assemblyPath): + if Path(assemblyPath).is_file(): with open(assemblyPath, "r") as raw: for line in raw: d = line.rstrip().split("\t") @@ -455,9 +450,9 @@ def main( useableOmes = dbOmes # initialize orthogroup data structures - if not hg_file and not os.path.isfile(wrk_dir + "homology_groups.tsv"): + if not hg_file and not Path(wrk_dir + "homology_groups.tsv").is_file(): hg_file = run_mmseqs(db, wrk_dir, algorithm=algorithm, min_id=min_id, cpus=cpus) - print("\tParsing homology groups (HGs)", flush=True) + logger.info("Parsing homology groups (HGs)") ome2i, gene2hg, i2ome, hg2gene = compile_homolog_groups( hg_file, wrk_dir, useableOmes ) @@ -468,9 +463,9 @@ def main( # remove genomes that are not in the db missing_from_db = set(ome2i.keys()).difference(set(db.keys())) - print("\t\tOmes:", len(ome2i), flush=True) + logger.info("%s %s", "\t\tOmes:", len(ome2i)) if missing_from_db: - print(f"\t\t\t{len(missing_from_db)} omes in HGs but not mtdb") + logger.debug(f"{len(missing_from_db)} omes in HGs but not mtdb") for ome in list(missing_from_db): del ome2i[ome] todel = [] @@ -489,32 +484,32 @@ def main( with open(wrk_dir + "ome2i.tsv", "w") as out: out.write("\n".join([k + "\t" + str(v) for k, v in ome2i.items()])) - print("\t\tHGs:", len(hg2gene), flush=True) - print("\t\tGenes:", len(gene2hg), flush=True) + logger.info("%s %s", "\t\tHGs:", len(hg2gene)) + logger.info("%s %s", "\t\tGenes:", len(gene2hg)) # compile cooccuring pairs of homogroups in each genome - print("\tCompiling all loci", flush=True) + logger.info("Compiling all loci") cc_arr_path = wrk_dir + "microsynt" ome2pairs = compile_loci(db, ome2i, gene2hg, plusminus * 2 + 1, cpus=cpus) cooccur_dict = None # if not os.path.isfile(out_dir + 'hgps.tsv.gz'): # assimilate cooccurrences across omes - print("\tIdentifying cooccurences", flush=True) + logger.info("Identifying cooccurences") seed_len = sum([len(ome2pairs[x]) for x in ome2pairs]) - print("\t\t" + str(seed_len) + " initial HG-pairs", flush=True) + logger.debug("" + str(seed_len) + " initial HG-pairs") cooccur_array, cooccur_dict, hgpair2i, i2hgpair = form_cooccur_structures( ome2pairs, 2, ome2i, cc_arr_path ) max_omes = max([len(cooccur_dict[x]) for x in cooccur_dict]) - print("\t\t" + str(max_omes) + " maximum organisms with HG-pair", flush=True) + logger.debug("" + str(max_omes) + " maximum organisms with HG-pair") cooccur_array[cooccur_array > 0] = 1 cooccur_array.astype(np.uint8) - print("\t\t" + str(sys.getsizeof(cooccur_array) / 1000000) + " MB", flush=True) + logger.debug("" + str(sys.getsizeof(cooccur_array) / 1000000) + " MB") cooccur_array, del_omes = remove_nulls(cooccur_array) for i in del_omes: - print(f"\t\t\t{i2ome[i]} removed for lacking overlap") + logger.debug(f"{i2ome[i]} removed for lacking overlap") del i2ome[i] ome2i = {v: i for i, v in enumerate(i2ome)} @@ -533,8 +528,8 @@ def main( ome2pairs = {ome2i[ome]: v for ome, v in ome2pairs.items() if ome in ome2i} microsynt_dict = {} - print("\nII. Microsynteny tree", flush=True) - if not os.path.isfile(tree_path): + logger.info("II. Microsynteny tree") + if not Path(tree_path).is_file(): nschgs = [] if near_single_copy_genes: try: @@ -550,11 +545,11 @@ def main( } ) # create microsynteny distance matrix and make tree - print("\tPreparing microsynteny alignment", flush=True) + logger.info("Preparing microsynteny alignment") align_file = align_microsynt_np( cooccur_array, i2ome, hg2gene, hgpair2i, wrk_dir, nschgs ) - print("\tBuilding microsynteny tree", flush=True) + logger.info("Building microsynteny tree") run_tree( align_file, wrk_dir, @@ -565,8 +560,8 @@ def main( cpus=cpus, ) # too bulky to justify keeping - if os.path.isfile(cc_arr_path + ".npy"): - os.remove(cc_arr_path + ".npy") + if Path(cc_arr_path + ".npy").is_file(): + Path(cc_arr_path + ".npy").unlink() return ome2i, gene2hg, i2ome, hg2gene, ome2pairs, cooccur_dict @@ -612,17 +607,18 @@ def cli(): parser.add_argument("-c", "--cpus", default=1, type=int) parser.add_argument("-o", "--output") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) execs = ["iqtree"] if args.orthofinder: of_out = format_path(args.orthofinder) - if os.path.isdir(of_out): + if Path(of_out).is_dir(): homogroups = of_out + "/Orthogroups/Orthogroups.txt" hg_dir = of_out + "/Orthogroup_Sequences/" else: homogroups = of_out - hg_dir = os.path.dirname(of_out) + "../Orthogroup_Sequences/" - if not os.path.isfile(hg_dir + "OG0000000.fa"): + hg_dir = str(Path(of_out).parent) + "../Orthogroup_Sequences/" + if not Path(hg_dir + "OG0000000.fa").is_file(): hg_dir = None method = "orthofinder" elif args.input: @@ -642,7 +638,7 @@ def cli(): findExecs(execs, exit=set(execs)) if not args.output: - out_dir = mkOutput(os.getcwd() + "/", "db2microsyntree") + out_dir = mkOutput(str(Path.cwd()) + "/", "db2microsyntree") else: out_dir = mkOutput(format_path(args.output), "db2microsyntree") @@ -658,8 +654,8 @@ def cli(): intro("db2microsyntree", args_dict, "Zachary Konkel") wrk_dir = out_dir + "working/" - if not os.path.isdir(wrk_dir): - os.mkdir(wrk_dir) + if not Path(wrk_dir).is_dir(): + Path(wrk_dir).mkdir() if args.focal_genes: with open(format_path(args.focal_genes), "r") as raw: diff --git a/mycotools/db2search.py b/mycotools/db2search.py index aa359d9..145825f 100755 --- a/mycotools/db2search.py +++ b/mycotools/db2search.py @@ -16,6 +16,7 @@ import re import sys import copy +import logging import datetime import argparse import subprocess @@ -30,12 +31,12 @@ multisub, findExecs, untardir, - eprint, format_path, mkOutput, tardir, inject_args, stdin2str, + setup_logging, ) from mycotools.lib.dbtools import primaryDB, mtdb from mycotools.lib.biotools import dict2fa, fa2dict, fa2dict_str @@ -44,6 +45,9 @@ from mycotools.acc2fa import dbmain as acc2fa_db, famain as acc2fa_fa from mycotools.utils.extractHmmsearch import main as exHmm from mycotools.utils.extractHmmAcc import grabAccs, main as absHmm +from pathlib import Path + +logger = logging.getLogger(__name__) def compile_hmm_cmd(db, hmm_path, output, ome_set=set(), cpu=1): @@ -100,13 +104,13 @@ def compileextractHmmCmd(db, args, output): def run_ex_hmm(args, hmmsearch_out, output): - ome = os.path.basename(hmmsearch_out).replace(".out", "") + ome = Path(hmmsearch_out).name.replace(".out", "") try: with open(hmmsearch_out, "r") as raw: data = raw.read() except FileNotFoundError: - eprint("\tWARNING: " + ome + " failed", flush=True) + logger.warning("\t" + ome + " failed") return ome, False if len(data) > 100: # check for data # check for data hmm_data = exHmm( @@ -129,7 +133,7 @@ def run_ex_hmm(args, hmmsearch_out, output): # does this overwrite other hits? return ome, out_dict else: - eprint("\tWARNING: " + ome + " empty results", flush=True) + logger.warning("\t" + ome + " empty results") return ome, False @@ -175,9 +179,9 @@ def compile_mafft_cmds(output, faa_dir): fas = collect_files(faa_dir, "faa") # grab completed fastas cmds = [] for fa in fas: - acc = os.path.basename(fa)[:-4] + acc = Path(fa).name[:-4] align = output + "/aligns/" + acc + ".mafft.fasta" - if os.path.isfile(align): # check for data + if Path(align).is_file(): # check for data with open(align, "r") as raw: data = raw.read() if len(data) > 10: @@ -196,7 +200,7 @@ def compile_hmmalign_cmds(output, accessions): align = output + "/aligns/" + acc + ".stockholm" conv = output + "/aligns/" + acc + ".phylip" trim = output + "/trimmed/" + acc + ".clipkit.fa" - if os.path.isfile(conv): + if Path(conv).is_file(): with open(conv, "r") as raw: data = raw.read() if len(data) > 10: @@ -233,14 +237,14 @@ def compile_trim_cmd(output, mod="", trimmed=None, ex="phylip"): cmd_tuples = [] aligns = collect_files(output + "aligns/", ex) if trimmed: - trimmed = set(os.path.basename(x).replace(".clipkit", "") for x in trimmed) + trimmed = set(Path(x).name.replace(".clipkit", "") for x in trimmed) aligns = [ - x for x in aligns if os.path.basename(x).replace(ex, "") not in trimmed + x for x in aligns if Path(x).name.replace(ex, "") not in trimmed ] for align in aligns: trim = ( f"{output}trimmed/" - + os.path.basename(align).replace(ex, "clipkit") + + Path(align).name.replace(ex, "clipkit") + "." + ex ) @@ -288,28 +292,28 @@ def hmmer_main( queries = compile_hmm_queries(hmm_paths, hmm_out) ome_set, skip1 = set(), False - if os.path.isdir(faa_dir): # is there a previous run? - print("\nCompiling previous run", flush=True) + if Path(faa_dir).is_dir(): # is there a previous run? + logger.info("Compiling previous run") # check if all fastas are made fas = collect_files(faa_dir, "faa") # grab completed fastas - ranQueries = [os.path.basename(fa).replace(".faa", "") for fa in fas] + ranQueries = [Path(fa).name.replace(".faa", "") for fa in fas] if not set(queries).difference(set(ranQueries)): skip1 = True if not skip1: # do not skip the first step - if os.path.isfile(output + "omes.tar.gz"): - if not os.path.isdir(ome_dir): + if Path(output + "omes.tar.gz").is_file(): + if not Path(ome_dir).is_dir(): untardir(output + "omes.tar.gz") # check what reports have been generated omes = collect_files(ome_dir, "out") - ome_set = set(os.path.basename(x).replace(".out", "") for x in omes) + ome_set = set(Path(x).name.replace(".out", "") for x in omes) else: - print("\thmmsearch -> hits.faa DONE", flush=True) + logger.info("\thmmsearch -> hits.faa DONE") if not skip1: - print("\nRunning " + os.path.basename(binary), flush=True) - if not os.path.isdir(ome_dir): - os.mkdir(ome_dir) + logger.info("Running " + Path(binary).name) + if not Path(ome_dir).is_dir(): + Path(ome_dir).mkdir() # run hmmer par_runs = round(((cpu - 1) / 2) - 0.5) @@ -326,17 +330,17 @@ def hmmer_main( ) for i, code in enumerate(hmmsearch_codes): if code: - eprint("\tERROR: " + str(hmmsearch_tuples[i]), flush=True) + logger.error("\t" + str(hmmsearch_tuples[i])) # extract results - print("\nExtracting hmmsearch output", flush=True) + logger.info("Extracting hmmsearch output") exHmm_args = [accessions, max_hits, query_cov, evalue, bitscore] exHmm_tuples = compileextractHmmCmd(db, exHmm_args, ome_dir) with mp.get_context("spawn").Pool(processes=cpu) as pool: hmmAligns = pool.starmap(run_ex_hmm, exHmm_tuples) mp.Process(target=tardir, args=[ome_dir]) - print("\nCompiling fastas", flush=True) + logger.info("Compiling fastas") # q_dict = {query: ome: alignment} q_dict = defaultdict(dict) for ome, hits in hmmAligns: @@ -414,8 +418,8 @@ def comp_diamond_tups( search_args=[], ): - if not os.path.isdir(out_dir + "dmnd/"): - os.mkdir(out_dir + "dmnd/") + if not Path(out_dir + "dmnd/").is_dir(): + Path(out_dir + "dmnd/").mkdir() blast_scaf = [ diamond, blast_type, @@ -480,8 +484,8 @@ def run_mmseq( ): db_dir = format_path("$MYCOGFF3/../db") - if not os.path.isdir(f"{out_dir}db/"): - os.mkdir(f"{out_dir}db/") + if not Path(f"{out_dir}db/").is_dir(): + Path(f"{out_dir}db/").mkdir() db_dir = format_path("$MYCOGFF3/../db/") createdb_cmds = [] db_path = f"{out_dir}db/searchdb" @@ -490,7 +494,7 @@ def run_mmseq( for i, ome in enumerate(seq_db["ome"]): db_path = f"{db_dir}{ome}_{biotype}" out_file = out_dir + ome + ".tsv" - if not os.path.isfile(db_path + ".dbtype"): + if not Path(db_path + ".dbtype").is_file(): # will fail at fastas that dont have sequences on one line createdb_cmds.append( ( @@ -506,7 +510,7 @@ def run_mmseq( ) if createdb_cmds: - print(f"\nCreating {len(createdb_cmds)} mmseqs search dbs", flush=True) + logger.info(f"Creating {len(createdb_cmds)} mmseqs search dbs") createdb_outs = multisub(createdb_cmds, processes=cpus, verbose=2) # if len(query) > 1: @@ -520,20 +524,20 @@ def run_mmseq( # query = [f'{out_dir}db/query'] # need to adjust check # create a concatenated mmseqs db for the search target - if not os.path.isfile(f"{out_dir}db/searchdb.dbtype"): + if not Path(f"{out_dir}db/searchdb.dbtype").is_file(): mergedbs_cmd = ["mmseqs", "mergedbs"] mergedbs_cmd.extend([f"{db_dir}{ome}_{biotype}" for ome in seq_db["ome"]]) mergedbs_cmd.insert(3, f"{out_dir}db/searchdb") - print("\nMerging search dbs", flush=True) + logger.info("Merging search dbs") mergedbs_out = subprocess.call(mergedbs_cmd) # , stderr = subprocess.DEVNULL, # stdout = subprocess.DEVNULL) - print("\nSearching", flush=True) + logger.info("Searching") for i, q in enumerate(query): out_file = f"{out_dir}{q}.tsv" - if os.path.isfile(out_file): + if Path(out_file).is_file(): continue - print("\t" + q, flush=True) + logger.info("\t" + q) search_cmd = [ mmseqs, "search", @@ -582,7 +586,7 @@ def parseOutput( ): ome_results = [ome, []] - if os.path.exists(file_): + if Path(file_).exists(): with open(file_, "r") as raw: data = [x.rstrip().split("\t") for x in raw if x.rstrip()] byq = defaultdict(list) @@ -609,7 +613,7 @@ def parseOutput_mmseqs( ): ome_results = [ome, []] - if os.path.exists(file_): + if Path(file_).exists(): with open(file_, "r") as raw: data = [x.rstrip().split("\t") for x in raw if x.rstrip()] byq = defaultdict(list) @@ -713,7 +717,7 @@ def comp_blast_acc2fa(db, biotype, output_res, coords=False, skip=None): try: acc2fa_cmds[query].append([list(set(accs)), db[i][biotype]]) except KeyError: - eprint("\t" + i + " not in db") + logger.warning("\t" + i + " not in db") return acc2fa_cmds @@ -723,23 +727,23 @@ def prepOutput(out_dir): out_dir = format_path(out_dir) if not out_dir.endswith("/"): out_dir += "/" - if not os.path.isdir(out_dir): - os.mkdir(out_dir) + if not Path(out_dir).is_dir(): + Path(out_dir).mkdir() report_dir = out_dir + "reports/" - if not os.path.isdir(report_dir): - os.mkdir(report_dir) + if not Path(report_dir).is_dir(): + Path(report_dir).mkdir() return report_dir def run_denovo(report_dir, log_list0, log_name): - if os.path.isdir(report_dir): + if Path(report_dir).is_dir(): count = 0 - while os.path.isdir(report_dir[:-1] + str(count)): + while Path(report_dir[:-1] + str(count)).is_dir(): count += 1 report_dir = report_dir[:-1] + str(count) + "/" log_list0[0] = report_dir - os.mkdir(report_dir) + Path(report_dir).mkdir() with open(log_name, "w") as out: out.write("\n".join(log_list0)) return log_list0, report_dir @@ -761,31 +765,31 @@ def db2searchLog( ] prev, reparse = False, False - log_name = out_dir + "." + os.path.basename(out_dir[:-1]) + ".log" + log_name = out_dir + "." + Path(out_dir[:-1]).name + ".log" log_list1 = None - if not os.path.isfile(log_name): # generate a new log + if not Path(log_name).is_file(): # generate a new log with open(log_name, "w") as out: out.write("\n".join(log_list0)) else: # check the old one with open(log_name, "r") as raw: log_list1 = [x.rstrip() for x in raw if x] if blast != log_list1[1]: - eprint("\tInconsistent search algorithm, rerunning", flush=True) + logger.warning("\tInconsistent search algorithm, rerunning") log_list0, report_dir = run_denovo(report_dir, log_list0, log_name) elif blast == "mmseqs": if log_list1[-1] != log_list0[-1]: # coverage is off, need a rerun - eprint("\tCoverage changed, rerunning", flush=True) + logger.warning("\tCoverage changed, rerunning") log_list0, report_dir = run_denovo(report_dir, log_list0, log_name) # need to reparse if anything is different elif any(log_list0[i] != log_list1[i] for i in range(len(log_list0))): - eprint("\tDeleting old report compilations", flush=True) + logger.info("\tDeleting old report compilations") reports = collect_files(report_dir, "tsv") for r in reports: - os.remove(r) + Path(r).unlink() reparse = True prev = True elif log_list1[1] != log_list0[1] and log_list1[3:] != log_list0[3:]: - eprint("\tInconsistent thresholds, rerunning", flush=True) + logger.warning("\tInconsistent thresholds, rerunning") log_list0, report_dir = run_denovo(report_dir, log_list0, log_name) else: prev = True @@ -807,7 +811,7 @@ def prepare_search_run( if prev: # reparse = False reports = collect_files(report_dir, "tsv") - finished = {os.path.basename(x)[:-4] for x in reports if os.path.getsize(x) > 0} + finished = {Path(x).name[:-4] for x in reports if Path(x).stat().st_size > 0} rundb, checkdb = mtdb({}).set_index("ome"), db.set_index("ome") for ome, val in checkdb.items(): if ome not in finished: @@ -821,10 +825,10 @@ def prepare_search_run( def prep_mmseq_output(rundb, report_dir, queries, convert=False): ome_res = defaultdict(str) for i, q in enumerate(queries): - if not os.path.isfile(f"{report_dir}{q}.tsv"): + if not Path(f"{report_dir}{q}.tsv").is_file(): continue with open(f"{report_dir}{q}.tsv", "r") as raw: - base = os.path.basename(q) + base = Path(q).name if not convert: for line in raw: line_d = line.split() @@ -845,7 +849,7 @@ def prep_mmseq_output(rundb, report_dir, queries, convert=False): def comp_mmseq_res(rundb, report_dir, queries, convert=False): for ome in rundb["ome"]: out_file = report_dir + ome + ".tsv" - if os.path.isfile(out_file): + if Path(out_file).is_file(): continue out_str, todel = "", [] for i, q in enumerate(queries): @@ -853,7 +857,7 @@ def comp_mmseq_res(rundb, report_dir, queries, convert=False): if not convert: out_str += raw.read().rstrip() + "\n" else: - base = os.path.basename(q) + base = Path(q).name for line in raw: line_d = line.split() line_d[0] = base @@ -863,7 +867,7 @@ def comp_mmseq_res(rundb, report_dir, queries, convert=False): out.write(out_str.rstrip()) for todel_file in todel: - os.remove(todel_file) + Path(todel_file).unlink() def ObyOsearch( @@ -887,7 +891,7 @@ def ObyOsearch( ppos=0, ): if len(rundb) > 0: - print("\nSearching on an ome-by-ome basis", flush=True) + logger.info("Searching on an ome-by-ome basis") if diamond: db_tups, search_tups = comp_diamond_tups( rundb, @@ -903,7 +907,7 @@ def ObyOsearch( search_args=search_arg, ) db_outs = multisub(db_tups, processes=cpus) - print(f"\t{len(search_tups)} searches to run", flush=True) + logger.info(f"\t{len(search_tups)} searches to run") search_outs = multisub( search_tups, processes=cpus, verbose=2, injectable=True ) @@ -944,7 +948,7 @@ def ObyOsearch( ] ) - print("\nParsing reports", flush=True) + logger.info("Parsing reports") with mp.get_context("spawn").Pool(processes=cpus) as pool: results = pool.starmap(parseOutput, tuple(parse_tups)) results_dict = {x[0]: x[1] for x in results} @@ -981,10 +985,10 @@ def mmseqs_mngr( cpus=cpus, ) - print("\nExtracting ome reports", flush=True) + logger.info("Extracting ome reports") prep_mmseq_output(rundb, report_dir, query, convert=convert) - print("\nParsing output", flush=True) + logger.info("Parsing output") # prepare report parsing commands for multiprocessing parse_tups = [] for i, ome in enumerate(db["ome"]): @@ -1010,24 +1014,24 @@ def mmseqs_mngr( def checkSearchDB(binary="blast"): - db_date = os.path.basename(primaryDB()) + db_date = Path(primaryDB()).name if "blast" in binary: search_db = format_path("$MYCOFAA/blastdb/" + db_date + ".00.psd") - if os.path.isfile(search_db): + if Path(search_db).is_file(): return search_db[:-7] - elif os.path.isfile(search_db[:-7] + ".psd"): + elif Path(search_db[:-7] + ".psd").is_file(): return search_db[:-7] else: search_db = format_path( "$MYCOFAA/blastdb/" + db_date.replace(".db", "") + ".mmseqs.db" ) - if os.path.isfile(search_db): + if Path(search_db).is_file(): return search_db def db_blast(db_path, blast_type, query, evalue, hsps, cpus, report_dir, diamond=False): - out_file = report_dir + os.path.basename(db_path)[:-3] + ".out" + out_file = report_dir + Path(db_path).name[:-3] + ".out" if not diamond: blast_scaf = [ blast_type, @@ -1085,7 +1089,7 @@ def db_blast(db_path, blast_type, query, evalue, hsps, cpus, report_dir, diamond def dbmmseq(db_path, query, evalue, cpus, report_dir, mmseqs="mmseqs", mem=None): out_file = ( - report_dir + os.path.basename(db_path)[:-3].replace(".mmseqs", "") + ".out" + report_dir + Path(db_path).name[:-3].replace(".mmseqs", "") + ".out" ) # output_str = '"query,target,pident,alen,mismatch,gapopen,qstart,qend,sstart,send,evalue,bits"' cmd_scaf = [ @@ -1202,7 +1206,7 @@ def mmseqs_main( reparse=reparse, ) - print("\nCompiling fastas", flush=True) + logger.info("Compiling fastas") output_res = compileResults(results_dict, skip) output_fas = {} acc2fa_cmds = comp_mmseq_acc2fa( @@ -1243,7 +1247,7 @@ def blast_main( seq_type = "nucl" biotype = "fna" else: - eprint("\nERROR: invalid search binary: " + blast, flush=True) + logger.error("invalid search binary: " + blast) # if blastdb: # insert function to make blastdb @@ -1264,12 +1268,12 @@ def blast_main( query = out_dir + "query.fa" if blastdb and not force: - print("\nSearching using MycotoolsDB searchdb", flush=True) + logger.info("Searching using MycotoolsDB searchdb") search_exit, search_output = db_blast( blastdb, blast, query, evalue, hsps, cpus, report_dir, diamond=diamond ) if search_exit: - eprint("\nERROR: search failed: " + str(search_exit)) + logger.error("search failed: " + str(search_exit)) sys.exit(10) results_dict = parseDBout( db, @@ -1314,7 +1318,7 @@ def blast_main( ppos=ppos, ) - print("\nCompiling fastas", flush=True) + logger.info("Compiling fastas") output_res = compileResults(results_dict, skip) output_fas = {} acc2fa_cmds = comp_blast_acc2fa( @@ -1323,7 +1327,7 @@ def blast_main( queryfa = fa2dict(query) for query1, cmd in acc2fa_cmds.items(): output_fas[query1] = {} - print("\t" + query1, flush=True) + logger.info("\t" + query1) with mp.get_context("spawn").Pool(processes=cpus) as pool: results = pool.starmap(acc2fa_fa, acc2fa_cmds[query1]) for x in results: @@ -1436,19 +1440,18 @@ def cli(): # parser.add_argument( '-c', '--coverage', type = float, help = 'Query coverage +/-, e.g. 0.5' ) # parser.add_argument('-f', '--force', action = 'store_true', help = 'Force ome-by-ome blast') args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.algorithm not in algorithms: - eprint("\nERROR: {args.algorithm} not implemented", flush=True) + logger.error("{args.algorithm} not implemented") sys.exit(19) # query parsing if not args.query and not args.query_dir and not args.query_file: - eprint( - "\nERROR: --query, --query_dir, or --query_file not specified", flush=True - ) + logger.error("--query, --query_dir, or --query_file not specified") sys.exit(18) elif [args.query, args.query_dir, args.query_file].count(True) > 1: - eprint("\nERROR: multiple query types specified", flush=True) + logger.error("multiple query types specified") sys.exit(20) else: if args.query: @@ -1459,7 +1462,7 @@ def cli(): elif args.query_dir: queries = [ format_path(args.query_dir) + x - for x in os.listdir(format_path(args.query_dir)) + for x in [p.name for p in Path(format_path(args.query_dir)).iterdir()] ] else: with open(format_path(args.query_file), "r") as raw: @@ -1469,7 +1472,7 @@ def cli(): # diamond specific if args.diamond: if "blast" not in args.algorithm: - eprint(f"\nERROR: --diamond incompatible with {args.algorithm}", flush=True) + logger.error(f"--diamond incompatible with {args.algorithm}") sys.exit(4) del deps[0] deps.append("diamond") @@ -1477,7 +1480,7 @@ def cli(): # mmseqs-specific if args.algorithm == "mmseqs": if not args.seqtype or args.seqtype not in {"aa", "nt"}: - eprint("\nERROR: -st required for mmseqs", flush=True) + logger.error("-st required for mmseqs") sys.exit(3) if args.seqtype == "aa": biotype = "faa" @@ -1498,17 +1501,17 @@ def cli(): # identity if args.identity: if args.algorithm == "hmmsearch": - eprint(f"\nERROR: --identity incompatible with hmmsearch", flush=True) + logger.error(f"--identity incompatible with hmmsearch") sys.exit(21) if not args.output: - base = os.getcwd() + "/" + base = str(Path.cwd()) + "/" output = mkOutput(base, "db2search") else: base = format_path(args.output, force_dir=True) output = base - if not os.path.isdir(output): - os.mkdir(output) + if not Path(output).is_dir(): + Path(output).mkdir() # output = mkOutput(base, 'db2search') if args.cpu and args.cpu < mp.cpu_count(): @@ -1598,8 +1601,8 @@ def cli(): convert=args.convert, iterations=args.iterations, ) - if not os.path.isdir(output + "fastas/"): - os.mkdir(output + "fastas/") + if not Path(output + "fastas/").is_dir(): + Path(output + "fastas/").mkdir() for query in output_fas: with open(output + "fastas/" + query + ".search.fa", "w") as out: diff --git a/mycotools/extract_mtdb.py b/mycotools/extract_mtdb.py index d8157bd..ccd7910 100755 --- a/mycotools/extract_mtdb.py +++ b/mycotools/extract_mtdb.py @@ -6,118 +6,21 @@ import os import re import sys -import copy -import random +import logging import argparse -from collections import defaultdict from mycotools.lib.kontools import ( file2list, intro, outro, format_path, - eprint, + setup_logging, mkOutput, ) from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.db2files import mtdb_main as gen_full_mtdb +from pathlib import Path -# NEED to fix when same lineage multiple ranks, e.g. Tremellales sp. will be listed -# as an order and as a genus - - -def infer_rank(db, lineage): - """Identify the taxonomic rank associated with an inputted lineage of - interest""" - linlow, rank = lineage.lower(), None - for ome, row in db.items(): - if linlow in set([x.lower() for x in row["taxonomy"].values()]): - rev_dict = { - k.lower(): v - for k, v in zip(row["taxonomy"].values(), row["taxonomy"].keys()) - } - rank = rev_dict[lineage] - - if not rank: - raise KeyError(f"no entry for {lineage}") - - return rank - - -def extract_unique(db, allowed=1, rank="species"): - """Extract unique rank from an MTDB""" - keys = copy.deepcopy(list(db.keys())) - random.shuffle(keys) - prep_db0 = {x: db[x] for x in keys} - prep_db1 = mtdb().set_index("ome") - if rank == "strain": - found = set() - for ome, row in prep_db0.items(): - name = row["taxonomy"]["species"] + " " + row["strain"] - if name not in found: - prep_db1[ome] = row - found_prep = list(found) - found_prep.append(name) - found = set(found_prep) - else: - found = defaultdict(int) - for ome, row in prep_db0.items(): - name = row["taxonomy"][rank] - found[name] += 1 - if found[name] <= allowed: - prep_db1[ome] = row - - return prep_db1 - - -def extract_tax(db, lineages): - """Extract specific taxonomic lineages of interest based on their rank""" - if isinstance(lineages, str): - lineages = [lineages] - lineages = set(x.lower() for x in lineages) - rank_dict = {k: infer_rank(db, k) for k in list(lineages)} - ranks = list(set(rank_dict.values())) - - new_db = mtdb().set_index() - for ome in db: - for rank in ranks: - try: - if db[ome]["taxonomy"][rank].lower() in lineages: - new_db[ome] = db[ome] - except KeyError: # invalid rank key for row - pass # probably should standardize tax jsons period - - return new_db - - -def extract_ome(db, omes, column="ome"): - """Extract a list of genome codes (omes) of interest""" - new_db = mtdb().set_index(column) - db = db.set_index(column) - for i in db: - if i in list(omes): - new_db[i] = db[i] - return new_db.set_index() - - -def extract_source(db, source): - """Extract an MTDB with genomes from a particular source""" - return mtdb( - { - ome: row - for ome, row in db.items() - if row["source"].lower() == source.lower() - }, - index="ome", - ) - - -def extract_pub(db): - """Extract only published and usable genomes""" - new_db = mtdb().set_index() - for ome, row in db.items(): - if row["published"]: - new_db[ome] = row - return new_db +logger = logging.getLogger(__name__) def main( @@ -136,27 +39,27 @@ def main( db = db.set_index("ome") if x_number > 0: - db = extract_unique(db, x_number, rank=rank) + db = db.extract_unique(x_number, rank=rank) # extract each taxonomic entry based on the classification specified if lineage_list: - new_db = extract_tax(db, lineage_list) + new_db = db.extract_tax(lineage_list) # if an ome list is specified then open it, store each entry in a list and pull each ome elif omes_set: - new_db = extract_ome(db, omes_set) + new_db = db.extract_ome(omes_set) elif aa_set: - new_db = extract_ome(db, aa_set, "assembly_acc") + new_db = db.extract_ome(aa_set, "assembly_acc") # if none of these are specified then create a `new_db` variable to work for later else: new_db = db # if there is a source specified, extract it or the opposite if source: - new_db = extract_source(new_db, source) + new_db = new_db.extract_source(source) # if you want publisheds, then just pull those out if not nonpublished: - new_db = extract_pub(new_db) + new_db = new_db.extract_pub() if inverse: new_omes = set(new_db.keys()) @@ -171,9 +74,9 @@ def main( dbs = {} for lineage in lineages: if lineage: - dbs[lineage.lower()] = extract_tax(new_db, [lineage]).reset_index() + dbs[lineage.lower()] = new_db.extract_tax([lineage]).reset_index() else: - dbs["unclassified"] = extract_tax(new_db, [""]).reset_index() + dbs["unclassified"] = new_db.extract_tax([""]).reset_index() return dbs else: return new_db.reset_index() @@ -238,27 +141,28 @@ def cli(): out_opt.add_argument("-o", "--output") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) db_path = format_path(args.mtdb) if args.lineage or args.lineages: - eprint( - "\nWARNING: extracting taxonomy is subject to " - + "errors in NCBI's hierarchy\n" + logger.warning( + "extracting taxonomy is subject to " + + "errors in NCBI's hierarchy" ) # these arguments require one another if args.by_rank and not args.rank: - eprint("\nERROR: --by_rank requires --rank", flush=True) + logger.error("--by_rank requires --rank") sys.exit(10) elif args.allowed_rank and not args.rank: - eprint("\nERROR: --allowed_rank requres --rank", flush=True) + logger.error("--allowed_rank requres --rank") sys.exit(11) elif args.rank and not args.allowed_rank and not args.by_rank: - eprint("\nERROR: --rank requires --allowed_rank or --by_rank", flush=True) + logger.error("--rank requires --allowed_rank or --by_rank") elif args.rank: args.rank = args.rank.lower() if args.rank not in set(ranks): - eprint(f"\nERROR: --rank not in {ranks}", flush=True) + logger.error(f"--rank not in {ranks}") sys.exit(12) args.lineage = args.lineage.replace('"', "").replace("'", "") @@ -277,7 +181,7 @@ def cli(): tag += args.source.lower() if not args.nonpublished: tag += "_pub" - output += "/" + os.path.basename(db_path) + tag + output += "/" + Path(db_path).name + tag if args.mtdb == "-": data = "" @@ -325,7 +229,7 @@ def cli(): new_db.df2db(output, paths=args.paths) else: out_dir = mkOutput(output, "extract_mtdb") - prefix = re.sub(r"\.mtdb$", "", os.path.basename(db_path)) + prefix = re.sub(r"\.mtdb$", "", Path(db_path).name) for lineage, db in new_db.items(): out_f = f"{out_dir}{prefix}.{lineage}.mtdb" db.df2db(out_f) diff --git a/mycotools/fa2clus.py b/mycotools/fa2clus.py index c1df052..4c6748a 100755 --- a/mycotools/fa2clus.py +++ b/mycotools/fa2clus.py @@ -5,7 +5,7 @@ # NEED to try MCL using the binary # NEED to make rerunning aggclus not use old data -import os +import logging import re import sys import copy @@ -22,14 +22,16 @@ multisub, findExecs, format_path, - eprint, - vprint, read_json, write_json, mkOutput, fmt_float, + setup_logging, ) from mycotools.lib.biotools import fa2dict, dict2fa +from pathlib import Path + +logger = logging.getLogger(__name__) sys.setrecursionlimit(1000000) @@ -53,7 +55,7 @@ def run_mmseqs( verbose=False, ): res_path = res_base + "_cluster.tsv" - tmp_dir = os.path.dirname(res_path) + "/tmp/" + tmp_dir = str(Path(res_path).parent) + "/tmp/" if verbose: stdout, stderr = None, None else: @@ -91,11 +93,11 @@ def run_mmseqs( raise ClusteringError( "Clustering failed: " + str(mmseqs_exit) + " " + " ".join(mmseqs_cmd) ) - if os.path.isfile(res_base + "_all_seqs.fasta"): - os.remove(res_base + "_all_seqs.fasta") - if os.path.isfile(res_base + "_rep_seq.fasta"): - os.remove(res_base + "_rep_seq.fasta") - if os.path.isdir(tmp_dir): + if Path(res_base + "_all_seqs.fasta").is_file(): + Path(res_base + "_all_seqs.fasta").unlink() + if Path(res_base + "_rep_seq.fasta").is_file(): + Path(res_base + "_rep_seq.fasta").unlink() + if Path(tmp_dir).is_dir(): shutil.rmtree(tmp_dir) return res_path @@ -125,7 +127,7 @@ def parse_mmseqs_clus(res_path): def makeDmndDB(diamond, queryFile, output_dir, cpus=1): """create a diamond database: diamond: binary path, queryFile: query_path""" - outputDB = output_dir + re.sub(r"\.[^\.]+$", "", os.path.basename(queryFile)) + outputDB = output_dir + re.sub(r"\.[^\.]+$", "", Path(queryFile).name) cmd = [ diamond, "makedb", @@ -395,7 +397,7 @@ def dmnd_main(fa_path, minVal, output_dir, distFile, pid=True, verbose=False, cp def usrch_main(fasta, min_id, output, cpus=1, verbose=False): - vprint("\nusearch aligning", flush=True, v=verbose) + logger.debug("usearch aligning") runUsearch(fasta, output + ".dist", str(1 - min_id), cpus, verbose) distance_matrix = rd_usrch_distmtx(output + ".dist") return distance_matrix @@ -415,8 +417,8 @@ def readLog(log_path, newLog): for i in newLog["successes"]: i["cluster"] = tuple(i["cluster"]) elif newLog["distance_matrix"]: - if os.path.isfile(newLog["distance_matrix"]): - os.remove(newLog["distance_matrix"]) + if Path(newLog["distance_matrix"]).is_file(): + Path(newLog["distance_matrix"]).unlink() return newLog @@ -470,7 +472,7 @@ def cluster_iter_mmseqs( ) res_info = res_base + name res_path = res_info + "_cluster.tsv" - if not os.path.isfile(res_path): + if not Path(res_path).is_file(): run_mmseqs( params["fa"], res_info, @@ -485,17 +487,13 @@ def cluster_iter_mmseqs( cluster = cluster_dict[clusters[focal_gene]] focal_len = len(cluster) - vprint( - "\nITERATION " + logger.debug("ITERATION " + str(attempt) + ": " + focal_gene + " cluster size: " - + str(focal_len), - flush=True, - v=verbose, - ) - vprint("Cluster parameter: " + str(clus_var), flush=True, v=verbose) + + str(focal_len)) + logger.debug("Cluster parameter: " + str(clus_var)) iteration_dict = { "size": focal_len, "cluster_variable": clus_var, @@ -506,7 +504,7 @@ def cluster_iter_mmseqs( if focal_len >= min_seq: # if greater than minimum sequences if max_seq: # if there is a max set of sequences if focal_len <= max_seq: # if less than max sequences - vprint(spacer + "\tSUCCESS!", flush=True, v=verbose) + logger.debug(spacer + "SUCCESS!") exit_code = 0 direction = -1 log_dict["successes"].append(iteration_dict) @@ -514,7 +512,7 @@ def cluster_iter_mmseqs( else: # descend direction = 1 else: # no max sequences, minimum is met, this was successful - vprint(spacer + "\tSUCCESS!", flush=True, v=verbose) + logger.debug(spacer + "SUCCESS!") exit_code = 0 log_dict["successes"].append(iteration_dict) log_dict["successes"] = sort_iterations(log_dict["successes"]) @@ -536,12 +534,9 @@ def cluster_iter_mmseqs( exit_code = 0 break else: - eprint( - spacer - + "WARNING: Overshot - " - + "could not find parameters using current interval", - flush=True, - ) + logger.warning(spacer + + "Overshot - " + + "could not find parameters using current interval") iteration = extract_closest_cluster( log_dict["iterations"], min_seq, max_seq ) @@ -626,17 +621,13 @@ def cluster_iter_aggclus( cluster = None newick = "" - vprint( - "\nITERATION " + logger.debug("ITERATION " + str(attempt) + ": " + focal_gene + " cluster size: " - + str(focal_len), - flush=True, - v=verbose, - ) - vprint("Cluster parameter: " + str(clus_var), flush=True, v=verbose) + + str(focal_len)) + logger.debug("Cluster parameter: " + str(clus_var)) iteration_dict = { "size": focal_len, "cluster_variable": clus_var, @@ -648,7 +639,7 @@ def cluster_iter_aggclus( if focal_len >= min_seq: # if greater than minimum sequences if max_seq: # if there is a max set of sequences if focal_len <= max_seq: # if less than max sequences - vprint(spacer + "\tSUCCESS!", flush=True, v=verbose) + logger.debug(spacer + "SUCCESS!") exit_code = 0 direction = -1 log_dict["successes"].append(iteration_dict) @@ -656,7 +647,7 @@ def cluster_iter_aggclus( else: # descend direction = 1 else: # no max sequences, minimum is met, this was successful - vprint(spacer + "\tSUCCESS!", flush=True, v=verbose) + logger.debug(spacer + "SUCCESS!") exit_code = 0 log_dict["successes"].append(iteration_dict) log_dict["successes"] = sort_iterations(log_dict["successes"]) @@ -673,12 +664,9 @@ def cluster_iter_aggclus( newick = log_dict["successes"][0]["tree"] exit_code = 0 else: - eprint( - spacer - + "WARNING: Overshot - " - + "could not find parameters using current interval", - flush=True, - ) + logger.warning(spacer + + "Overshot - " + + "could not find parameters using current interval") iteration = extract_closest_cluster( log_dict["iterations"], min_seq, max_seq ) @@ -730,8 +718,8 @@ def main( max_var=1, ): - if not os.path.isdir(os.path.dirname(output) + "/working/"): - os.mkdir(os.path.dirname(output) + "/working/") + if not Path(str(Path(output).parent) + "/working/").is_dir(): + Path(str(Path(output).parent) + "/working/").mkdir() if search_program in {"usearch", "diamond"}: algorithm = "hierarchical" @@ -756,7 +744,7 @@ def main( "successes": [], } if log_path: - if os.path.isfile(log_path): + if Path(log_path).is_file(): log_dict = readLog(log_path, log_dict) write_json(log_dict, log_path) @@ -769,9 +757,9 @@ def main( param_dict = { "dist": None, "link": linkage, - "dir": os.path.dirname(output) + "/", + "dir": str(Path(output).parent) + "/", } - if os.path.isfile(log_dict["distance_matrix"]): + if Path(log_dict["distance_matrix"]).is_file(): if search_program == "usearch": param_dict["dist"] = rd_usrch_distmtx(log_dict["distance_matrix"]) else: @@ -781,7 +769,7 @@ def main( else: if search_program == "diamond": # elif if above lines not highlighted if not dmnd_dir: - dmnd_dir = os.path.dirname(log_dict["distance_matrix"]) + "/" + dmnd_dir = str(Path(log_dict["distance_matrix"]).parent) + "/" param_dict["dist"] = dmnd_main( fa_path, clus_const, @@ -800,11 +788,11 @@ def main( "fa": fa_path, "bin": search_program, "clus_const": clus_const, - "dir": os.path.dirname(output) + "/", + "dir": str(Path(output).parent) + "/", } if focal_gene: - vprint("\nClustering", flush=True, v=verbose) + logger.debug("Clustering") if algorithm == "hierarchical": cluster, newick, log_dict, error = cluster_iter_aggclus( param_dict, @@ -854,7 +842,7 @@ def main( res_base = param_dict["dir"] + focal_gene else: res_base = param_dict["dir"] + re.sub( - r"\.[^\.]+$", "", os.path.basename(fa_path) + r"\.[^\.]+$", "", Path(fa_path).name ) if algorithm == "hierarchical": clusters, tree = scipyaggd( @@ -970,6 +958,7 @@ def cli(): parser.add_argument("-c", "--cpus", default=1, type=int) parser.add_argument("-v", "--verbose", action="store_true") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) # if args.refine and not args.max_seq: # eprint('\nERROR: --max_seq required for refinement', flush = True) @@ -981,10 +970,10 @@ def cli(): "centroid", "single", }: - eprint("\nERROR: Invalid linkage criterium", flush=True) + logger.error("Invalid linkage criterium") sys.exit(1) elif args.distance_type not in {"identity", "bitscore"}: - eprint("\nERROR: Invalid distance type", flush=True) + logger.error("Invalid distance type") sys.exit(3) if args.distance_type == "identity": pid = True @@ -1007,13 +996,13 @@ def cli(): if not args.cluster_variable: args.cluster_variable = 0.3 else: - eprint("\nERROR: Invalid alignment software", flush=True) + logger.error("Invalid alignment software") sys.exit(2) findExecs([args.alignment.split()[0]], exit=set(args.alignment.split()[0])) interval = args.interval if args.interval < 0 or args.interval > 1: - eprint("\nERROR: --interval must be between 0 and 1", flush=True) + logger.error("--interval must be between 0 and 1") # if args.iterative and args.alignment in {'diamond', 'usearch'}: # clus_var = 1 - args.cluster_constant @@ -1021,10 +1010,7 @@ def cli(): 1 - args.cluster_variable <= args.cluster_constant and args.alignment != "mmseqs" ): - eprint( - "\nWARNING: 1 - maximum distance exceeds minimum connection, clustering is ineffective", - flush=True, - ) + logger.warning("1 - maximum distance exceeds minimum connection, clustering is ineffective") sys.exit(3) else: clus_var = args.cluster_variable @@ -1035,20 +1021,20 @@ def cli(): fa = fa2dict(fa_path) if args.iterative: if len(fa) < args.min_seq: - eprint("\nERROR: minimum sequences is greater than fasta input", flush=True) + logger.error("minimum sequences is greater than fasta input") sys.exit(5) elif args.iterative not in fa: - eprint("\nERROR: " + args.iterative + " not in " + fa_path, flush=True) + logger.error("" + args.iterative + " not in " + fa_path) sys.exit(6) if args.output: - if not os.path.isdir(format_path(args.output)): - os.mkdir(format_path(args.output)) + if not Path(format_path(args.output)).is_dir(): + Path(format_path(args.output)).mkdir() dmnd_dir = format_path(args.output) - output = dmnd_dir + re.sub(r"\.[^\.]+$", "", os.path.basename(fa_path)) + output = dmnd_dir + re.sub(r"\.[^\.]+$", "", Path(fa_path).name) else: - dmnd_dir = mkOutput(os.getcwd() + "/", "fa2clus") - output = dmnd_dir + re.sub(r"\.[^\.]+$", "", os.path.basename(fa_path)) + dmnd_dir = mkOutput(str(Path.cwd()) + "/", "fa2clus") + output = dmnd_dir + re.sub(r"\.[^\.]+$", "", Path(fa_path).name) cluster, tree, overshot, log_dict = main( fa_path, @@ -1062,7 +1048,7 @@ def cli(): focal_gene=args.iterative, interval=interval, output=output, - log_path=dmnd_dir + "." + os.path.basename(fa_path) + ".fa2clus.json", + log_path=dmnd_dir + "." + Path(fa_path).name + ".fa2clus.json", pid=pid, dmnd_dir=dmnd_dir, cpus=args.cpus, @@ -1077,7 +1063,7 @@ def cli(): try: output_fa = {x: input_fa[x] for x in cluster} except TypeError: - eprint("\nERROR: empty cluster", flush=True) + logger.error("empty cluster") sys.exit(10) with open(output + ".fa", "w") as out: out.write(dict2fa(output_fa)) diff --git a/mycotools/fa2hmmer2fa.py b/mycotools/fa2hmmer2fa.py index 6b294a6..e58ca6c 100755 --- a/mycotools/fa2hmmer2fa.py +++ b/mycotools/fa2hmmer2fa.py @@ -2,6 +2,7 @@ # NEED to ditch extracthmm and move to simplified output parsing +import logging import os import re import sys @@ -9,13 +10,16 @@ import datetime import subprocess import multiprocessing as mp -from mycotools.extractHmmsearch import main as exHmm, grabNames -from mycotools.extractHmmAcc import main as extr_hmm +from mycotools.utils.extractHmmsearch import main as exHmm, grab_names as grabNames +from mycotools.utils.extractHmmAcc import main as extr_hmm from mycotools.db2search import compAcc2fa from mycotools.acc2fa import famain as acc2fa -from mycotools.lib.kontools import intro, outro, findExecs, eprint, format_path +from mycotools.lib.kontools import intro, outro, findExecs, format_path, setup_logging from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.biotools import dict2fa +from pathlib import Path + +logger = logging.getLogger(__name__) def runextractHmmAcc(hmm, accession, output): @@ -80,7 +84,7 @@ def run_acc2fa(db, biotype, output_res, subhit=True, cpu=1): acc2fa_cmds = compAcc2fa(db, biotype, output_res, subhit) output_fas = {} for query in acc2fa_cmds: - print("\t" + query, flush=True) + logger.debug("" + query) with mp.get_context("spawn").Pool(processes=cpu) as pool: results = pool.starmap(acc2fa, acc2fa_cmds[query]) output_fas[query] = "\n".join([dict2fa(x) for x in results]) @@ -116,12 +120,12 @@ def main( biotype = "faa" if accession: - print("\nExtracting " + accession, flush=True) + logger.debug("Extracting " + accession) hmm_path = runextractHmmAcc(hmm_path, accession, out_dir + accession + ".hmm") - if os.path.isfile(accession): + if Path(accession).is_file(): accession = [] else: - print("\nExtracting accessions", flush=True) + logger.info("Extracting accessions") with open(hmm_path, "r") as raw: hmm_data = raw.read() accession = [] @@ -131,18 +135,18 @@ def main( else: hmm_cpu = cpu hmmer_out = out_dir + "hmmer.out" - print("\nRunning " + binary, flush=True) + logger.debug("Running " + binary) if runHmmer(fasta_path, hmm_path, hmmer_out, cpu=hmm_cpu, binary=binary): - eprint("\tERROR: " + binary + " failed", flush=True) + logger.error("" + binary + " failed") sys.exit(2) - print("\nParsing output", flush=True) + logger.info("Parsing output") hmm_data = run_extract_hmm( hmmer_out, top_hits, cov_threshold, evalue, not accession_search, accession ) output_res = parse_hmm_data(hmm_data) - print("\nCompiling fastas", flush=True) + logger.info("Compiling fastas") output_fas = run_acc2fa(db, biotype, output_res, subhit=subhit, cpu=cpu) return output_fas @@ -180,9 +184,10 @@ def cli(): parser.add_argument("-o", "--output", help="Output directory") parser.add_argument("--cpu", default=1, type=int) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.binary not in {"hmmsearch", "nhmmer"}: - eprint("\nERROR: invalid --binary", flush=True) + logger.error("invalid --binary") sys.exit(1) findExecs([args.binary], exit={args.binary}) @@ -190,9 +195,9 @@ def cli(): out_dir = args.output else: date = datetime.datetime.today().strftime("%Y%m%d") - out_dir = os.getcwd() + "/" + date + "_fa2hmm2fa/" - if not os.path.isdir(out_dir): - os.mkdir(out_dir) + out_dir = str(Path.cwd()) + "/" + date + "_fa2hmm2fa/" + if not Path(out_dir).is_dir(): + Path(out_dir).mkdir() out_dir = format_path(out_dir) if args.evalue: @@ -235,7 +240,7 @@ def cli(): subhit=not args.whole, ) fastaname = re.sub( - r"\.fa[^\.]*$", "", os.path.basename(os.path.abspath(args.fasta)) + r"\.fa[^\.]*$", "", Path(os.path.abspath(args.fasta)).name ) outputFas(output_fas, out_dir, fastaname) diff --git a/mycotools/fa2mass.py b/mycotools/fa2mass.py index ad096ad..b39fd57 100755 --- a/mycotools/fa2mass.py +++ b/mycotools/fa2mass.py @@ -1,14 +1,18 @@ #! /usr/bin/env python3 -import os import re import sys -from mycotools.lib.kontools import sys_start, format_path, fmt_float +import logging +from mycotools.lib.kontools import sys_start, format_path, fmt_float, setup_logging from mycotools.lib.biotools import fa2dict, calc_weight +logger = logging.getLogger(__name__) + + def cli(): + setup_logging() usage = "USAGE: Inputs amino acid fasta outputs linear protein weights" args = sys_start(sys.argv, usage, 1) diff --git a/mycotools/fa2tree.py b/mycotools/fa2tree.py index b6c53e8..0c51538 100755 --- a/mycotools/fa2tree.py +++ b/mycotools/fa2tree.py @@ -4,6 +4,7 @@ # NEED to ignore non fasta inputs # NEED to work as a standalone script +import logging import os import re import sys @@ -14,8 +15,6 @@ import multiprocessing as mp from collections import defaultdict from mycotools.lib.kontools import ( - eprint, - vprint, collect_files, format_path, intro, @@ -24,20 +23,22 @@ mkOutput, multisub, parse_run_log, + setup_logging, ) from mycotools.lib.biotools import fa2dict, dict2fa +from pathlib import Path + +logger = logging.getLogger(__name__) try: from clipkit import clipkit except ImportError: - eprint("ERROR: clipkit is not installed. Install via `conda` or `pip`") + logger.error("clipkit is not installed. Install via `conda` or `pip`") try: from ete3 import Tree except ImportError: - eprint( - "WARNING: ete3 not installed.\nInstall ete3 into your " - + "conda environment via `conda install ete3`" - ) + logger.warning("ete3 not installed.\nInstall ete3 into your " + + "conda environment via `conda install ete3`") # adopted from https://stackoverflow.com/a/2829036 for verbosity control @@ -68,7 +69,7 @@ def run_mafft( # run the command directly if not hpc: - print(spacer + "Aligning", flush=True) + logger.debug(spacer + "Aligning") with open(out_dir + name + ".mafft", "w") as out_file: if verbose: run_mafft = subprocess.call(cmd, stdout=out_file) @@ -78,9 +79,9 @@ def run_mafft( ) if run_mafft != 0: - eprint(spacer + "\tERROR: mafft failed: " + str(run_mafft), flush=True) - if os.path.isfile(out_dir + name + ".mafft"): - os.remove(out_dir + name + ".mafft") + logger.error(spacer + "mafft failed: " + str(run_mafft)) + if Path(out_dir + name + ".mafft").is_file(): + Path(out_dir + name + ".mafft").unlink() if pass_fail: return None raise PhyloError @@ -128,7 +129,7 @@ def run_clipkit( ): """Execute ClipKIT from a complete Mafft run""" - clipkit_out_name = out_dir + os.path.basename(mafft_name) + ".clipkit" + clipkit_out_name = out_dir + Path(mafft_name).name + ".clipkit" if gappy: mode = "gappy" cmd = [ @@ -147,7 +148,7 @@ def run_clipkit( # execute immediately if not hpc: - print(spacer + "Trimming", flush=True) + logger.debug(spacer + "Trimming") if not verbose: clipkit_code = subprocess.call(cmd, stdout=subprocess.PIPE) else: @@ -176,10 +177,8 @@ def run_clipkit( # ) # no output file, the run failed - if not os.path.isfile(clipkit_out_name): - eprint( - spacer + "\tERROR: `clipkit` failed: " + str(clipkit_code), flush=True - ) + if not Path(clipkit_out_name).is_file(): + logger.error(spacer + "`clipkit` failed: " + str(clipkit_code)) if pass_fail: return None raise PhyloError @@ -246,10 +245,10 @@ def run_mf( "-s", f_, "--prefix", - out_dir + os.path.basename(f_), + out_dir + Path(f_).name, ] for f_ in clipkit_files - if not os.path.isfile(f"{out_dir}{os.path.basename(f_)}.contree") + if not Path(f"{out_dir}{Path(f_).name}.contree").is_file() ] # append the topological constraint to each command if it is present if constraint: @@ -260,9 +259,9 @@ def run_mf( clipkit_files = [ x for x in clipkit_files - if not os.path.isfile(f"{out_dir}{os.path.basename(x)}.contree") + if not Path(f"{out_dir}{Path(x).name}.contree").is_file() ] - return {os.path.basename(v): cmds[i] for i, v in enumerate(clipkit_files)} + return {Path(v).name: cmds[i] for i, v in enumerate(clipkit_files)} # otherwise parallelize and run ModelFinder directly multisub(cmds, verbose=verbose, processes=concurrent_cmds) @@ -270,7 +269,7 @@ def run_mf( # parse and identify the evolutionary models determined by ModelFinder models = {} for f_ in clipkit_files: - with open(out_dir + os.path.basename(f_) + ".log", "r") as raw: + with open(out_dir + Path(f_).name + ".log", "r") as raw: for line in raw: if line.startswith("Best-fit model:"): model = re.search(r"Best-fit model: (.+) chosen", line)[1] @@ -293,12 +292,9 @@ def prepare_nexus(concat_fa, models, spacer="\t"): # if some alignments aren't the same length then there is some cryptic # issue, likely user-caused if not all(len(x["sequence"]) == len0 for x in trim_fa.values()): - eprint( - spacer - + "\tERROR: alignment sequences are not same length " - + trimmed_f, - flush=True, - ) + logger.error(spacer + + "alignment sequences are not same length " + + trimmed_f) sys.exit(17) # adjust the index of the coordinates of each sequence based on the # previous sequences' length @@ -356,11 +352,11 @@ def run_partition_tree( if constraint: cmd.extend(["-g", constraint]) if tree_stop: - print(spacer + "Concatenated nexus and fasta outputted", flush=True) - print(" ".join(cmd), flush=True) + logger.debug(spacer + "Concatenated nexus and fasta outputted") + logger.debug(" ".join(cmd)) sys.exit(0) - print(spacer + "Tree building", flush=True) + logger.debug(spacer + "Tree building") if verbose: run_tree = subprocess.call(cmd) else: @@ -384,7 +380,7 @@ def run_tree_reconstruction( """Manage and execute phylogeny reconstruction from an inputted ClipKIT output trimmed alignment.""" - tree_file = out_dir + os.path.basename(clipkit_file) + tree_file = out_dir + Path(clipkit_file).name # prepare a fasttree ommand if fast: cmd = ["fasttree", "-out", tree_file + ".treefile", clipkit_file] @@ -409,7 +405,7 @@ def run_tree_reconstruction( # execute in the current terminal if not hpc: - print(spacer + "Tree building", flush=True) + logger.debug(spacer + "Tree building") if fast: cmd[2] += ".tmp" if verbose: @@ -421,12 +417,12 @@ def run_tree_reconstruction( if fast: shutil.move(cmd[2], cmd[2][:-4]) if run_tree != 0: - eprint(spacer + "\tERROR: tree failed: " + str(run_tree), flush=True) + logger.error(spacer + "tree failed: " + str(run_tree)) raise PhyloError # prepare an .sh file for user execution, or sequential execution from the # previous scripts else: - vprint("\nOutputting bash script `tree.sh`.\n", v=verbose, flush=True) + logger.debug("Outputting bash script `tree.sh`.\n") with open(f"{out_dir}../{prefix}_tree.sh", "w") as out: out.write(hpc + "\n\n" + " ".join([str(x) for x in cmd])) @@ -435,37 +431,37 @@ def prep_fasta_path_input(fasta_path, output_dir): """Prepare the output directories and collect the files from an input that is a fasta path (file or directory)""" # start from a file input - if os.path.isfile(fasta_path): + if Path(fasta_path).is_file(): if not output_dir: dir_name = re.sub( - r"\..*?$", "_tree", os.path.basename(os.path.abspath(fasta_path)) + r"\..*?$", "_tree", Path(os.path.abspath(fasta_path)).name ) out_dir = output_dir + "/" + dir_name - if not os.path.isdir(out_dir): - os.mkdir(out_dir) + if not Path(out_dir).is_dir(): + Path(out_dir).mkdir() out_dir = format_path(out_dir) else: - if not os.path.isdir(output_dir): - os.mkdir(output_dir) + if not Path(output_dir).is_dir(): + Path(output_dir).mkdir() out_dir = format_path(output_dir) wrk_dir = out_dir + "working/" - if not os.path.isdir(wrk_dir): - os.mkdir(wrk_dir) + if not Path(wrk_dir).is_dir(): + Path(wrk_dir).mkdir() files = [fasta_path] # start from a directory of fastas - elif os.path.isdir(fasta_path): + elif Path(fasta_path).is_dir(): if not output_dir: out_dir = mkOutput(output_dir_prep, "fa2tree") else: - if not os.path.isdir(output_dir): - os.mkdir(output_dir) + if not Path(output_dir).is_dir(): + Path(output_dir).mkdir() out_dir = format_path(output_dir) wrk_dir = out_dir + "working/" - if not os.path.isdir(wrk_dir): - os.mkdir(wrk_dir) + if not Path(wrk_dir).is_dir(): + Path(wrk_dir).mkdir() files = collect_files(fasta_path, "*") check_fas = set(files) @@ -475,11 +471,11 @@ def prep_fasta_path_input(fasta_path, output_dir): if f + ".mafft" not in check_fas: new_fas.append(f) else: - shutil.copy(f + ".mafft", wrk_dir + os.path.basename(f) + ".mafft") + shutil.copy(f + ".mafft", wrk_dir + Path(f).name + ".mafft") else: shutil.copy( f + ".mafft.clipkit", - wrk_dir + os.path.basename(f) + ".mafft.clipkit", + wrk_dir + Path(f).name + ".mafft.clipkit", ) files = new_fas return out_dir, wrk_dir, files @@ -489,15 +485,15 @@ def prep_fasta_list_input(fastas, output_dir): if not output_dir: out_dir = mkOutput("./", "fa2tree") else: - if not os.path.isdir(output_dir): - os.mkdir(output_dir) + if not Path(output_dir).is_dir(): + Path(output_dir).mkdir() out_dir = format_path(output_dir) wrk_dir = out_dir + "working/" - if not os.path.isdir(wrk_dir): - os.mkdir(wrk_dir) + if not Path(wrk_dir).is_dir(): + Path(wrk_dir).mkdir() files = [] for path in fastas: - if os.path.isdir(path): + if Path(path).is_dir(): files.extend([format_path(x) for x in collect_files(path, "*")]) else: files.append(path) @@ -509,10 +505,8 @@ def nonfasta2fasta(files, conv_dir): from Bio import SeqIO for i, f_ in enumerate(files): - out_name = conv_dir + os.path.basename( - re.search(r"(.*)\.[^.]+$", f_)[1] + ".fa" - ) - if not os.path.isfile(out_name): + out_name = conv_dir + Path(re.search(r"(.*)\.[^.]+$", f_)[1] + ".fa").name + if not Path(out_name).is_file(): if f_.endswith( ( ".nexus", @@ -555,21 +549,18 @@ def identify_incomplete_files(files, flag_incomplete, wrk_dir): if f.endswith("/"): f = f[:-1] fa_dict = fa2dict(f) - incomp_omes[os.path.basename(f)] = [] + incomp_omes[Path(f).name] = [] # populate a hash with each gene for each ome associated with each # fasta for seq in fa_dict: ome = seq[: seq.find("_")] if f not in ome2fa2gene[ome]: - incomp_omes[os.path.basename(f)].append(ome) + incomp_omes[Path(f).name].append(ome) ome2fa2gene[ome][f] = seq # a multigene partition cannot be reconstructed for a genome with # multiple genes in the same alignment else: - eprint( - "\nERROR: multiple sequences for a " + "single ome in " + f, - flush=True, - ) + logger.error("multiple sequences for a " + "single ome in " + f) sys.exit(5) # identify the files with missing genomes and the missing omes themselves @@ -591,23 +582,19 @@ def identify_incomplete_files(files, flag_incomplete, wrk_dir): # it is an error if genomes are missing and it isn't explicitly # permitted if flag_incomplete: - eprint( - "\nERROR: omes without sequences in all fastas\n", - +"Run with -m to remove failed omes", - flush=True, + logger.error( + "omes without sequences in all fastas\n" + "Run with -m to remove failed omes" ) else: - eprint( - "\nWARNING: omes removed without sequences in all \ - fastas: ", - flush=True, - ) + logger.warning("omes removed without sequences in all \ + fastas: ") for f, omes in incomp_files.items(): - eprint("\t" + f + ": " + ",".join([x for x in omes]), flush=True) + logger.info("" + f + ": " + ",".join([x for x in omes])) if comp_files: - eprint("\tComplete files: " + ",".join(comp_files), flush=True) + logger.info("Complete files: " + ",".join(comp_files)) else: - eprint("\tNo complete files", flush=True) + logger.info("No complete files") # exit if incomplete flags if flag_incomplete: sys.exit(6) @@ -616,14 +603,14 @@ def identify_incomplete_files(files, flag_incomplete, wrk_dir): else: new_files = [] for fasta in files: - fasta_name = os.path.basename(fasta) + fasta_name = Path(fasta).name fa = fa2dict(fasta) new_fa = { k: v for k, v in fa.items() if k[: k.find("_")] not in del_omes } with open(wrk_dir + fasta_name + ".tmp", "w") as out: out.write(dict2fa(new_fa)) - os.rename(wrk_dir + fasta_name + ".tmp", wrk_dir + fasta_name) + Path(wrk_dir + fasta_name + ".tmp").rename(wrk_dir + fasta_name) new_files.append(wrk_dir + fasta_name) files = new_files @@ -638,7 +625,7 @@ def algn_mngr( trimmed_files = [] for fasta in files: name = re.search(r".*?/*([^/]+)/*$", fasta)[1] - fasta_name = os.path.basename(os.path.abspath(fasta)) + fasta_name = Path(os.path.abspath(fasta)).name clipkit = wrk_dir + fasta_name + ".clipkit" # if the starting point is a non-aligned fasta @@ -647,8 +634,8 @@ def algn_mngr( clipkit = mafft + ".clipkit" try: # check if it exists - if os.stat(mafft): - vprint("\nAlignment exists", v=verbose, flush=True) + if Path(mafft).stat(): + logger.debug("Alignment exists") else: raise ValueError except (FileNotFoundError, ValueError) as e: @@ -677,8 +664,8 @@ def algn_mngr( if start == 0 or start == 1: try: # check for a trimmed fasta - if os.stat(clipkit): - vprint("\nTrim exists", v=verbose, flush=True) + if Path(clipkit).stat(): + logger.debug("Trim exists") clipkit_out = clipkit else: raise ValueError @@ -706,7 +693,7 @@ def convert_seq_to_ome_name(trimmed_files, conv_dir): """Convert the inputted sequence name to its genome name""" new_trimmed_files = [] for trimmed_f in trimmed_files: - new_f = conv_dir + os.path.basename(trimmed_f) + new_f = conv_dir + Path(trimmed_f).name in_fa = fa2dict(trimmed_f) out_fa = {} for seq, data in in_fa.items(): @@ -724,20 +711,17 @@ def extract_supported(trimmed_files, min_mean_support, out_dir): """Extract trees that meet the minimum summary support value""" out_files = [] for f_ in trimmed_files: - t_path = f"{out_dir}working/{os.path.basename(f_)}.contree" + t_path = f"{out_dir}working/{Path(f_).name}.contree" supports = check_tree_support(t_path) if supports is None: continue mean_support = sum(supports) / len(supports) if mean_support >= min_mean_support: - print(f"\t{os.path.basename(t_path)} {mean_support} passed", flush=True) + logger.debug(f"{Path(t_path).name} {mean_support} passed") out_files.append(f_) else: - print(f"\t{os.path.basename(t_path)} {mean_support} failed", flush=True) - print( - f"\t{len(out_files)} ({len(out_files)/len(trimmed_files)*100}%)" + " passed", - flush=True, - ) + logger.debug(f"{Path(t_path).name} {mean_support} failed") + logger.debug(f"{len(out_files)} ({len(out_files)/len(trimmed_files)*100}%)" + " passed") return out_files @@ -781,21 +765,21 @@ def multigene_mngr( + f"#SBATCH --job-name={f}_tree\n" ) out.write(" ".join(cmd)) - print(spacer + "Alignments outputed", flush=True) + logger.debug(spacer + "Alignments outputed") sys.exit(0) - print(spacer + "Model finding", flush=True) + logger.debug(spacer + "Model finding") models = run_mf( trimmed_files, wrk_dir, constraint, verbose, cpus, spacer=spacer + "\t" ) - print(f"{spacer}\t{len(models)} individual trees completed", flush=True) + logger.debug(f"{spacer}\t{len(models)} individual trees completed") if min_mean_support: - print(spacer + "Extracting passing trees", flush=True) + logger.debug(spacer + "Extracting passing trees") passing = extract_supported(trimmed_files, min_mean_support, out_dir) models = {k: models[k] for k in passing} # build the concatenated NEXUS - print(spacer + "Concatenating", flush=True) + logger.debug(spacer + "Concatenating") empty_concat_fa = { ome: {"description": "", "sequence": ""} for ome in ome2fa2gene @@ -872,9 +856,7 @@ def main( # prepare the scaffold of an HPC command if that is requested hpc, hpc_prep = False, False if slurm: - vprint( - "\nHPC mode, preparing submission scripts (Slurm)\n", v=verbose, flush=True - ) + logger.debug("HPC mode, preparing submission scripts (Slurm)\n") hpc_prep = ( "#!/bin/bash\n#SBATCH --time=24:00:00\n" + "#SBATCH --nodes=1\n#SBATCH --ntasks-per-node=" @@ -883,16 +865,14 @@ def main( ) # '\n\nsource activate ' + source elif torque: - vprint( - "\nHPC mode, preparing submission scripts (PBS)\n", v=verbose, flush=True - ) + logger.debug("HPC mode, preparing submission scripts (PBS)\n") hpc_prep = ( "#PBS -l walltime=10:00:00\n#PBS -l nodes=1:ppn=4\n#PBS " + "-A " + project ) if not partition and hpc_prep: hpc = hpc_prep - output_dir_prep = os.getcwd() + "/" + output_dir_prep = str(Path.cwd()) + "/" # the fasta data should be a path if it is a string if isinstance(fasta_path, str): out_dir, wrk_dir, files = prep_fasta_path_input(fasta_path, output_dir) @@ -902,8 +882,8 @@ def main( # create a directory for converting files conv_dir = wrk_dir + "conv/" - if not os.path.isdir(conv_dir): - os.mkdir(conv_dir) + if not Path(conv_dir).is_dir(): + Path(conv_dir).mkdir() # check for non fasta inputs - if these files exist in a directory then # they will be used anyway @@ -922,7 +902,7 @@ def main( ) for f_ in files ): - print(spacer + "Converting to fastas", flush=True) + logger.debug(spacer + "Converting to fastas") files = nonfasta2fasta(files, conv_dir) # only proceed with fastas from the inputted files @@ -978,12 +958,9 @@ def main( if hpc: if slurm: - print("\nStart pipeline via `sbatch .sh` in " + out_dir + "\n") + logger.debug("Start pipeline via `sbatch .sh` in " + out_dir + "\n") else: - vprint( - "\nStart pipeline via `qsub .sh` in " + out_dir + "\n", - v=verbose, - ) + logger.debug("Start pipeline via `qsub .sh` in " + out_dir + "\n") def cli(): @@ -1067,6 +1044,7 @@ def cli(): r_opt.add_argument("-c", "--cpus", default=1, type=int) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) execs = ["mafft", "clipkit"] if args.fast: @@ -1080,7 +1058,7 @@ def cli(): if args.support: if args.support > 1 or args.support < 0: - eprint("\nERROR: --support must be between 0 and 1") + logger.error("--support must be between 0 and 1") sys.exit(3) output = format_path(args.output) @@ -1090,7 +1068,7 @@ def cli(): if args.gappy: if args.gappy > 1: - eprint("\nERROR: gappy threshold must be less than 1") + logger.error("gappy threshold must be less than 1") sys.exit(3) args_dict = { diff --git a/mycotools/fna2faa.py b/mycotools/fna2faa.py index 8670141..126e98b 100755 --- a/mycotools/fna2faa.py +++ b/mycotools/fna2faa.py @@ -1,12 +1,17 @@ #! /usr/bin/env python3 import sys +import logging from Bio.Seq import Seq -from mycotools.lib.kontools import format_path, stdin2str, sys_start +from mycotools.lib.kontools import format_path, stdin2str, sys_start, setup_logging from mycotools.lib.biotools import fa2dict, dict2fa +logger = logging.getLogger(__name__) + + def cli(): + setup_logging() usage = 'Input nucleotide fasta ("-" for stdin), translate to protein fasta' args = sys_start(sys.argv[1:], usage, 1) if args[0] == "-": diff --git a/mycotools/gff2seq.py b/mycotools/gff2seq.py index c230d2b..acff629 100755 --- a/mycotools/gff2seq.py +++ b/mycotools/gff2seq.py @@ -1,12 +1,15 @@ #! /usr/bin/env python3 +import logging import re import sys import argparse from Bio.Seq import Seq from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.biotools import fa2dict, gff2list, gff3Comps, dict2fa -from mycotools.lib.kontools import format_path, sys_start, eprint, stdin2str +from mycotools.lib.kontools import format_path, sys_start, stdin2str, setup_logging + +logger = logging.getLogger(__name__) def sortGene(sorting_group): @@ -98,9 +101,7 @@ def grabCDS(gff_dicts, spacer="\t"): genes.extend(alias.split("|")) except TypeError: if not warning and ome: - eprint( - spacer + "WARNING: " + str(ome) + " missing aliases", flush=True - ) + logger.warning(spacer + "" + str(ome) + " missing aliases") warning = True raise TypeError(str(entry)) @@ -659,6 +660,7 @@ def cli(): "-af", "--all_flanks", action="store_true", help="-n and -nc only" ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.gff == "-": data = stdin2str() @@ -680,9 +682,7 @@ def cli(): assembly_dicts[ome] = fa2dict(db[ome]["fna"]) gff_dicts[ome].append(line) except IndexError: - eprint( - "\nERROR: " + args.gff + " is incompatible with MycotoolsDB", flush=True - ) + logger.error("" + args.gff + " is incompatible with MycotoolsDB") sys.exit(1) if args.protein: diff --git a/mycotools/gff2svg.py b/mycotools/gff2svg.py index 24db53e..db28893 100755 --- a/mycotools/gff2svg.py +++ b/mycotools/gff2svg.py @@ -3,14 +3,24 @@ # NEED TO PARSE FOR IN GENE COORDINATES AND ANNOTATIONS # NEED to create a single file output option for multiple inputs -import os import re import sys import random +import logging import argparse -from mycotools.lib.kontools import sys_start, format_path, file2list, getColors +from mycotools.lib.kontools import ( + sys_start, + format_path, + file2list, + getColors, + setup_logging, +) from dna_features_viewer import GraphicFeature, GraphicRecord from mycotools.lib.biotools import gff2list, gff3Comps +from pathlib import Path + + +logger = logging.getLogger(__name__) def compileProducts(gff, prod_comp, types={"tRNA", "mRNA", "rRNA"}): @@ -61,7 +71,7 @@ def gff2svg( elif not gen_new_colors: product = null elif product not in product_dict: - print(product) + logger.debug(product) try: # try to use the next color product_dict[product] = colors[count] count += 1 @@ -207,6 +217,7 @@ def cli(): ) parser.add_argument("-o", "--output", help="Optional output directory") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if not args.regex: regex = gff3Comps()["product"] @@ -230,13 +241,13 @@ def cli(): if args.output: out_dir = format_path(args.output) else: - out_dir = format_path(os.path.dirname(args.input)) + out_dir = format_path(str(Path(args.input).parent)) gffs = file2list(args.input) for entry in gffs: if entry.endswith("/"): entry = re.sub(r"/+$", "", entry) - svg_path = os.path.dirname(gffs[0]) + re.sub( - r"\.gf[^\.]+$", ".svg", os.path.basename(gffs[0]) + svg_path = str(Path(gffs[0]).parent) + re.sub( + r"\.gf[^\.]+$", ".svg", Path(gffs[0]).name ) product_dict = main( gff2list(gffs[0]), @@ -248,8 +259,8 @@ def cli(): shuffle=args.shuffle, ) for gff in gffs[1:]: - svg_path = os.path.dirname(gff) + re.sub( - r"\.gf[^\.]+$", ".svg", os.path.basename(gff) + svg_path = str(Path(gff).parent) + re.sub( + r"\.gf[^\.]+$", ".svg", Path(gff).name ) product_dict = main( gff2list(gff), @@ -267,11 +278,11 @@ def cli(): args.gff = re.sub(r"/+$", "", args.gff) if args.output: svg_path = format_path(args.output) + re.sub( - r"\.gf[^\.]+$", ".svg", os.path.basename(args.gff) + r"\.gf[^\.]+$", ".svg", Path(args.gff).name ) else: - svg_path = os.path.dirname(args.gff) + re.sub( - r"\.gf[^\.]+$", ".svg", os.path.basename(args.gff) + svg_path = str(Path(args.gff).parent) + re.sub( + r"\.gf[^\.]+$", ".svg", Path(args.gff).name ) main( gff2list(args.gff), diff --git a/mycotools/jgiDwnld.py b/mycotools/jgiDwnld.py index 63fb703..aaea565 100755 --- a/mycotools/jgiDwnld.py +++ b/mycotools/jgiDwnld.py @@ -11,21 +11,25 @@ import re import sys import time +import logging import getpass import argparse import subprocess import pandas as pd import xml.etree.ElementTree as ET from tqdm import tqdm -from mycotools.lib.kontools import eprint, format_path, outro, intro +from mycotools.lib.kontools import format_path, outro, intro, setup_logging from mycotools.lib.dbtools import loginCheck +from pathlib import Path + +logger = logging.getLogger(__name__) def jgi_login(user, pwd): """Login via JGI's prescribed method by creating a cookie cache and downloading JGI's sign-in file.""" - null = os.path.expanduser("~/.nulljgi_dwnld") + null = str(Path("~/.nulljgi_dwnld").expanduser()) login_cmd = subprocess.call( [ @@ -49,7 +53,7 @@ def jgi_login(user, pwd): def dwnld_xml(output, ome, max_tempts=2): attempts = 0 - while not os.path.isfile(f"{output}/{ome}.xml") and attempts < max_tempts: + while not Path(f"{output}/{ome}.xml").is_file() and attempts < max_tempts: attempts += 1 xml_cmd = subprocess.call( [ @@ -65,8 +69,8 @@ def dwnld_xml(output, ome, max_tempts=2): stderr=subprocess.PIPE, ) if xml_cmd != 0: - print(f"\tERROR: {ome} xml curl error: {xml_cmd}", flush=True) - if not os.path.isfile(f"{output}/{ome}.xml"): + logger.error(f"\t{ome} xml curl error: {xml_cmd}") + if not Path(f"{output}/{ome}.xml").is_file(): return -1 else: return xml_cmd @@ -77,16 +81,16 @@ def retrieve_xml(ome, output): download it using JGI's prescribed method. Then open the xml and check for the common 'Portal does not exist' error. If so, report.""" - if os.path.exists(output + "/" + str(ome) + ".xml"): + if Path(output + "/" + str(ome) + ".xml").exists(): with open(output + "/" + str(ome) + ".xml", "r") as xml_raw: xml_data = xml_raw.read() if xml_data == "Portal does not exist": - print("\tERROR: `" + ome + " not in JGIs `organism` database", flush=True) + logger.error("\t`" + ome + " not in JGIs `organism` database") xml_cmd = 1 - os.remove(output + "/" + ome + ".xml") + Path(output + "/" + ome + ".xml").unlink() elif not xml_data: xml_cmd = None - os.remove(f"{output}/{ome}.xml") + Path(f"{output}/{ome}.xml").unlink() else: xml_cmd = -1 else: @@ -96,12 +100,12 @@ def retrieve_xml(ome, output): with open(output + "/" + ome + ".xml", "r") as xml_raw: xml_data = xml_raw.read() if xml_data == "Portal does not exist": - print("\tERROR: `" + ome + " not in JGIs `organism` database", flush=True) + logger.error("\t`" + ome + " not in JGIs `organism` database") xml_cmd = 1 - os.remove(output + "/" + ome + ".xml") + Path(output + "/" + ome + ".xml").unlink() elif not xml_data: xml_cmd = None - os.remove(f"{output}/{ome}.xml") + Path(f"{output}/{ome}.xml").unlink() return xml_cmd @@ -267,9 +271,8 @@ def handle_redirect_307( ): """Handle a redirection error by identifying a new file URL to download from, or return the original if none exist""" - print( - spacer + "\t" + dwnld + " link has moved. " + "Trying a different link.", - flush=True, + logger.info( + spacer + "\t" + dwnld + " link has moved. " + "Trying a different link." ) filename, n_url, dwnld_md5, t_org_name = parse_xml( file_type, xml_file, masked=masked, forbidden={url}.union(urls) @@ -278,7 +281,7 @@ def handle_redirect_307( if n_url: url = n_url dwnld_url = prefix + url.replace("&", "&") - dwnld = f"{output}{file_type}/{os.path.basename(dwnld_url)}" + dwnld = f"{output}{file_type}/{Path(dwnld_url).name}" return url, dwnld_url, dwnld, {url}.union(urls), t_org_name @@ -286,11 +289,11 @@ def no_md5_checks(dwnld, md5, spacer): """If there is no MD5, simply check the file has content in it""" check_size = subprocess.run(["wc", "-l", dwnld], stdout=subprocess.PIPE) check_size_res = check_size.stdout.decode("utf-8") - print(spacer + "\t\tFile exists - no md5 to check.", flush=True) + logger.info(spacer + "\t\tFile exists - no md5 to check.") check_size_find = re.search(r"\d+", check_size_res) size = check_size_find[0] if int(size) < 10: - print(spacer + "\tInvalid file size.", flush=True) + logger.warning(spacer + "\tInvalid file size.") else: md5 = None return md5 @@ -348,17 +351,17 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): dwnld_url = prefix + url.replace("&", "&") - dwnld = f"{output}{file_type}/{os.path.basename(dwnld_url)}" + dwnld = f"{output}{file_type}/{Path(dwnld_url).name}" unzip_dwnld = re.sub(r"\.gz$", "", dwnld) # assume unzipped downloads have passed the checks - if os.path.isfile(unzip_dwnld): + if Path(unzip_dwnld).is_file(): md5 = dwnld_md5 curl_cmd = 0 check = unzip_dwnld preexisting = True # if the file currently exists, then check its MD5 - elif os.path.exists(dwnld): + elif Path(dwnld).exists(): if dwnld_md5: md5_cmd = subprocess.run( ["md5sum", dwnld], stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -404,7 +407,7 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): preexisting = True check = dwnld else: - print(spacer + "\tmd5 does not match.", flush=True) + logger.warning(spacer + "\tmd5 does not match.") break # while the MD5 doesn't match, or there is a curl error, try up to 3 @@ -434,7 +437,7 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): md5 = md5_find[0] except TypeError: md5 = False - if not os.path.isfile(dwnld): + if not Path(dwnld).is_file(): attempt += 1 continue # if there is no MD5 attempt the crude file check @@ -460,7 +463,7 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): org_name = t_org_name if t_url == url: - print(spacer + "\t\tNo valid alternative", flush=True) + logger.warning(spacer + "\t\tNo valid alternative") attempt = 4 break else: @@ -478,10 +481,9 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): # the download may have failed, so prepare to retry elif md5 != dwnld_md5 and attempt == 1: - print( - f"{spacer}\tERROR: md5 does not match JGI. " - + f"Attempt {attempt}", - flush=True, + logger.error( + f"{spacer}\tmd5 does not match JGI. " + + f"Attempt {attempt}" ) curl_cmd = -1 check = 2 @@ -506,8 +508,8 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): if t_org_name: org_name = t_org_name if t_url == url: - print( - spacer + "\t\tNo valid alternative", flush=True + logger.warning( + spacer + "\t\tNo valid alternative" ) attempt = 4 break @@ -521,9 +523,8 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): time.sleep(60) # if there are two fails, attempt a new URL elif md5 != dwnld_md5 and attempt == 2: - print( - f"{spacer}\tERROR: md5 does not match JGI. Attempt {attempt}", - flush=True, + logger.error( + f"{spacer}\tmd5 does not match JGI. Attempt {attempt}" ) curl_cmd = -1 filename, n_url, dwnld_md5, t_org_name = parse_xml( @@ -536,32 +537,30 @@ def jgi_dwnld(ome, file_type, output, masked=True, spacer="\t"): url = n_url dwnld_url = prefix + url.replace("&", "&") f_ulrs = {url}.union(f_urls) - dwnld = f"{output}{file_type}/{os.path.basename(dwnld_url)}" + dwnld = f"{output}{file_type}/{Path(dwnld_url).name}" time.sleep(60) check = 2 elif md5 != dwnld_md5: - print( - f"{spacer}\tERROR: md5 does not match JGI. Attempt {attempt}", - flush=True, + logger.error( + f"{spacer}\tmd5 does not match JGI. Attempt {attempt}" ) check = 2 else: - print( - f"{spacer}\tERROR: Failed to retrieve {file_type}. `curl` error: " - + f"{curl_cmd}\n{spacer}\tAttempt {attempt}", - flush=True, + logger.error( + f"{spacer}\tFailed to retrieve {file_type}. `curl` error: " + + f"{curl_cmd}\n{spacer}\tAttempt {attempt}" ) check = 2 # three strikes and the file is out if attempt == 3: if md5 != dwnld_md5: - print( - spacer + "\tExcluding from database - potential failure", flush=True + logger.warning( + spacer + "\tExcluding from database - potential failure" ) curl_cmd = 0 if curl_cmd != 0: - print(spacer + "\tFile failed to download", flush=True) + logger.error(spacer + "\tFile failed to download") return check, preexisting, file_type, ran_dwnld, org_name @@ -582,31 +581,30 @@ def main( # pd.options.mode.chained_assignment = None # default='warn' if not "assembly_acc" in df.columns: if len(df.columns) != 1: - eprint( - "\nInvalid input. No assembly_acc column and more than one column.", - flush=True, + logger.error( + "Invalid input. No assembly_acc column and more than one column." ) else: ome_col = list(df.columns)[0] else: ome_col = "assembly_acc" - eprint(spacer + "Logging into JGI", flush=True) + logger.info(spacer + "Logging into JGI") login_attempt = 0 while jgi_login(user, pwd) != 0 and login_attempt < 5: - eprint( - spacer + "\tJGI Login Failed. Attempt: " + str(login_attempt), flush=True + logger.warning( + spacer + "\tJGI Login Failed. Attempt: " + str(login_attempt) ) time.sleep(5) login_attempt += 1 if login_attempt == 3: - eprint(spacer + "\tERROR: Failed 3 login attempts.", flush=True) + logger.error(spacer + "\tFailed 3 login attempts.") sys.exit(100) - if not os.path.exists(output + "/xml"): - os.mkdir(output + "/xml") + if not Path(output + "/xml").exists(): + Path(output + "/xml").mkdir() # perhaps add a counter here, but one that checks if it is actually querying jgi - print("\nRetrieving `xml` directories", flush=True) + logger.info("Retrieving `xml` directories") ome_set, count = set(), 0 for i, row in tqdm(df.iterrows(), total=len(df)): error_check, attempt = True, 0 @@ -621,12 +619,11 @@ def main( elif error_check != -1: time.sleep(0.3) if error_check != -1: - eprint(f"{spacer}\t{row[ome_col]} failed to retrieve XML", flush=True) + logger.warning(f"{spacer}\t{row[ome_col]} failed to retrieve XML") ome_set.add(row[ome_col]) - eprint( - f"{spacer}Downloading {len(df)} JGI files\n\t" + "Maximum rate: 1 file/min", - flush=True, + logger.info( + f"{spacer}Downloading {len(df)} JGI files\n\t" + "Maximum rate: 1 file/min" ) dwnlds = [] @@ -643,8 +640,8 @@ def main( dwnlds.append("est") for typ in dwnlds: - if not os.path.isdir(output + "/" + typ): - os.mkdir(output + "/" + typ) + if not Path(output + "/" + typ).is_dir(): + Path(output + "/" + typ).mkdir() preexisting, ran_dwnld = True, False for i, row in df.iterrows(): @@ -654,18 +651,18 @@ def main( if ome not in ome_set: jgi_login(user, pwd) if "ome" in row.keys(): - eprint(spacer + row["ome"] + "\t" + ome, flush=True) + logger.info(spacer + row["ome"] + "\t" + ome) else: - eprint(spacer + ome, flush=True) + logger.info(spacer + ome) for typ in dwnlds: check, preexisting, new_typ, ran_dwnld, org_name = jgi_dwnld( ome, typ, output, masked=masked, spacer=spacer ) if type(check) != int: df.at[i, new_typ + "_path"] = ( - output + "/" + new_typ + "/" + os.path.basename(check) + output + "/" + new_typ + "/" + Path(check).name ) - check = os.path.basename(os.path.abspath(check)) + check = Path(os.path.abspath(check)).name if org_name: org_d = org_name.split() genus = org_d[0] @@ -684,16 +681,16 @@ def main( df.at[i, "strain"] = strain elif type(check) == int: ome_set.add(row[ome_col]) - eprint( - spacer + "\t" + new_typ + ": exit status " + str(check), flush=True + logger.info( + spacer + "\t" + new_typ + ": exit status " + str(check) ) else: - eprint(spacer + ome + " failed.", flush=True) + logger.warning(spacer + ome + " failed.") - if os.path.exists("cookies"): - os.remove("cookies") - if os.path.exists(os.path.expanduser("~/.null")): - os.remove(os.path.expanduser("~/.null")) + if Path("cookies").exists(): + Path("cookies").unlink() + if Path(str(Path("~/.null").expanduser())).exists(): + Path(str(Path("~/.null").expanduser())).unlink() if "gff3" in df.columns: del df["gff3"] @@ -753,8 +750,9 @@ def cli(): action="store_true", help="[-a] Download nonmasked assemblies", ) - parser.add_argument("-o", "--output", default=os.getcwd(), help="Output dir") + parser.add_argument("-o", "--output", default=str(Path.cwd()), help="Output dir") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.nonmasked: args.assembly = True @@ -766,7 +764,7 @@ def cli(): and not args.est and not args.gff ): - eprint("\nERROR: You must choose at least one download option.", flush=True) + logger.error("You must choose at least one download option.") ncbi_email, ncbi_api, user, pwd = loginCheck(ncbi=False) # user = input( 'JGI username: ' ) @@ -783,16 +781,15 @@ def cli(): } start_time = intro("Download JGI files", args_dict) - eprint( - "\nWARNING: This script does NOT account for use-restricted data. " + logger.warning( + "This script does NOT account for use-restricted data. " + "It is user responsibility to determine use restriction status " + "in accord with the MycoCosm terms and conditions: " - + "https://jgi.doe.gov/user-programs/pmo-overview/policies/legacy-data-policies/", - flush=True, + + "https://jgi.doe.gov/user-programs/pmo-overview/policies/legacy-data-policies/" ) - eprint(flush=True) + logger.info("") - if os.path.isfile(args.input): + if Path(args.input).is_file(): with open(args.input, "r") as raw: for line in raw: if "assembly_acc" in line.rstrip().split("\t"): @@ -822,7 +819,7 @@ def cli(): jgi_df = jgi_df.rename(columns={"assembly_acc": "#assembly_acc"}) jgi_df["source"] = "jgi" jgi_df["restriction"] = "no" - jgi_df.to_csv(os.path.normpath(args.input) + ".predb.tsv", sep="\t", index=False) + jgi_df.to_csv(str(Path(args.input)) + ".predb.tsv", sep="\t", index=False) outro(start_time) diff --git a/mycotools/lib/biotools.py b/mycotools/lib/biotools.py index fdfbccf..cae4e0f 100755 --- a/mycotools/lib/biotools.py +++ b/mycotools/lib/biotools.py @@ -6,7 +6,6 @@ import sys from collections import defaultdict from itertools import chain -from mycotools.lib.kontools import eprint aa_weights = { "A": 89.1, diff --git a/mycotools/lib/dbtools.py b/mycotools/lib/dbtools.py index dc8793e..e3b43f6 100755 --- a/mycotools/lib/dbtools.py +++ b/mycotools/lib/dbtools.py @@ -2,11 +2,14 @@ # NEED to make login check more intuitive and easier for NCBI only +from __future__ import annotations + import os import re import sys import copy import json +import logging import time import base64 import urllib @@ -14,17 +17,21 @@ import zipfile import datetime import subprocess +import random from tqdm import tqdm from Bio import Entrez from io import StringIO +from typing import Any, Dict, Iterable, Mapping, Optional, Union from collections import defaultdict from mycotools.lib.kontools import ( collect_files, - eprint, format_path, read_json, write_json, ) +from pathlib import Path + +logger = logging.getLogger(__name__) class mtdb(dict): @@ -50,24 +57,28 @@ class mtdb(dict): "gff3", ] - # NEED a detect index feature for adding dicts in with alternative indices - def __init__(self, db=None, index=None, add_paths=True): - self.columns = [ - "ome", + #: columns that may serve as the MTDB index (unique, hashable keys) + _index_columns = frozenset({"assembly_acc", "ome"}) + #: taxonomic ranks excluded when assimilating taxonomy dictionaries + _forbidden_tax_ranks = frozenset( + { + "no rank", + "subkingdom", "genus", "species", - "strain", - "taxonomy", - "version", - "source", - "biosample", - "assembly_acc", - "acquisition_date", - "published", - "fna", - "faa", - "gff3", - ] + "species group", + "varietas", + "forma", + } + ) + + # NEED a detect index feature for adding dicts in with alternative indices + def __init__( + self, + db: Union[str, Mapping[str, Any], "mtdb", None] = None, + index: Optional[str] = None, + add_paths: bool = True, + ): if not db: if not index: super().__init__({x: [] for x in self.columns}) @@ -90,17 +101,18 @@ def __init__(self, db=None, index=None, add_paths=True): ): super().__init__(db) else: - super().__init__(mtdb.db2df(self, db, add_paths=add_paths)) + super().__init__(self.db2df(db, add_paths=add_paths)) self.index = index - def mtdb2pd(self): + def mtdb2pd(self) -> "pd.DataFrame": import pandas as pd copy_mtdb = copy.deepcopy(self) copy_mtdb = copy_mtdb.reset_index() return pd.DataFrame(copy_mtdb) # assume pd is imported - def pd2mtdb(df): # legacy integration + @staticmethod + def pd2mtdb(df: "pd.DataFrame") -> "mtdb": # legacy integration df = df.fillna("") for i, row in df.iterrows(): if not row["gff3"]: @@ -112,9 +124,9 @@ def pd2mtdb(df): # legacy integration db = mtdb({x: list(df[x]) for x in mtdb.columns}) return db - def db2df(self, db_path, add_paths=True): + def db2df(self, db_path: str, add_paths: bool = True) -> Dict[str, list]: df = defaultdict(list) - if os.stat(db_path).st_size == 0: + if Path(db_path).stat().st_size == 0: return {x: [] for x in mtdb.columns} with open(format_path(db_path), "r") as raw: data = [ @@ -129,9 +141,9 @@ def db2df(self, db_path, add_paths=True): for i, d in enumerate(entry): df[columns[i]][-1] = d try: - df["taxonomy"][-1] = read_tax(df["taxonomy"][-1]) + df["taxonomy"][-1] = self.read_tax(df["taxonomy"][-1]) except json.decoder.JSONDecodeError: - print(df["taxonomy"][-1]) + logger.error("malformed taxonomy: %s", df["taxonomy"][-1]) sys.exit() df["taxonomy"][-1]["genus"] = df["genus"][-1] df["taxonomy"][-1]["species"] = df["genus"][-1] + " " + df["species"][-1] @@ -164,14 +176,16 @@ def db2df(self, db_path, add_paths=True): df["faa"][i] = os.environ["MYCOFAA"] + ome + ".faa" df["gff3"][i] = os.environ["MYCOGFF3"] + ome + ".gff3" except KeyError: - eprint( - "ERROR: MycotoolsDB not in path, cannot delineate biofile paths", - flush=True, - ) + logger.error("MycotoolsDB not in path, cannot delineate biofile paths") return df - def df2db(self, db_path=None, headers=False, paths=False): + def df2db( + self, + db_path: Optional[str] = None, + headers: bool = False, + paths: bool = False, + ) -> None: df = copy.copy(self) df = df.reset_index() output = mtdb( @@ -237,7 +251,7 @@ def df2db(self, db_path=None, headers=False, paths=False): flush=True, ) - def set_index(self, column="ome", inplace=False): + def set_index(self, column: Optional[str] = "ome", inplace: bool = False) -> "mtdb": data, retry, error, df, columns = ( {}, bool(column), @@ -247,7 +261,7 @@ def set_index(self, column="ome", inplace=False): ) if not column: return df.reset_index() - elif column not in {"assembly_acc", "ome"}: + elif column not in self._index_columns: raise KeyError(f'MTDB index must be "assembly_acc"/"ome"') elif df.index and not df.keys(): # empty df return mtdb({}, index=column) @@ -288,7 +302,7 @@ def set_index(self, column="ome", inplace=False): data[v][-1][head] = df[head][i] except KeyError: # if an index exists if error: - eprint("\nERROR: invalid column", flush=True) + logger.error("invalid column") return self df = df.reset_index() error = True @@ -297,7 +311,7 @@ def set_index(self, column="ome", inplace=False): retry = False return mtdb(data, column) - def reset_index(self): + def reset_index(self) -> "mtdb": df = copy.copy(self) if df.index: data = {x: [] for x in mtdb().columns} @@ -315,9 +329,11 @@ def reset_index(self): else: return df - def append(self, info={}): + def append(self, info: Optional[Mapping[str, Any]] = None) -> "mtdb": # if any(x not in set(self.columns) for x in info): # raise KeyError('Invalid keys: ' + str(set(info.keys()).difference(set(self.columns)))) + if info is None: + info = {} index = self.index df = copy.copy(self) df = df.reset_index() @@ -329,6 +345,179 @@ def append(self, info={}): df[key].append(info[key]) return df.set_index(index) + @staticmethod + def read_tax( + taxonomy_string: Union[str, Mapping[str, Any], None] + ) -> Dict[str, Any]: + """Read taxonomy from an MTDB by converting the string into a dictionary""" + tax_strs = [ + "superkingdom", + "kingdom", + "phylum", + "subphylum", + "class", + "order", + "family", + "subfamily", + ] + if taxonomy_string: + if isinstance(taxonomy_string, str): + dict_string = taxonomy_string.replace("'", '"') + try: + tax_dict = json.loads(dict_string) + except TypeError: + tax_dict = {} + else: + tax_dict = taxonomy_string + try: + tax_dict = {**tax_dict, **{x: "" for x in tax_strs if x not in tax_dict}} + except TypeError: # inappropriate tax_dict in the column + tax_dict = {x: "" for x in tax_strs} + return tax_dict + else: + return {} + + @staticmethod + def _reconcile_tax_dicts( + genera: Iterable[str], + tax_dicts: Mapping[str, Mapping[str, Any]], + forbid: Iterable[str], + ) -> Dict[str, Dict[str, Any]]: + """Drop empty/forbidden taxonomy entries and backfill missing genera""" + forbid = set(forbid) + tax_dicts = {x: tax_dicts[x] for x in tax_dicts if tax_dicts[x]} + for genus in tax_dicts: + tax_dicts[genus] = { + rank: name for rank, name in tax_dicts[genus].items() if rank not in forbid + } + for miss in set(genera).difference(tax_dicts.keys()): + tax_dicts[miss] = {} + return tax_dicts + + def prepare_tax_dicts( + self, tax_dicts: Optional[Dict[str, Any]] = None + ) -> "tuple[set, Dict[str, Any]]": + """Identify the genera that do not have higher taxonomy ascribed to them""" + if tax_dicts is None: + tax_dicts = {} + need_tax = set() + for ome, entry in self.set_index("ome").items(): + if entry["genus"] in tax_dicts: + continue + tax_json = self.read_tax(entry["taxonomy"]) + if any( + name + for rank, name in tax_json.items() + if rank not in {"genus", "species", "strain"} + ): + tax_dicts[entry["genus"]] = tax_json + else: + need_tax.add(entry["genus"]) + need_tax = need_tax.difference(tax_dicts.keys()) + return need_tax, tax_dicts + + def assimilate_tax( + self, + tax_dicts: Mapping[str, Mapping[str, Any]], + forbid: Optional[Iterable[str]] = None, + ) -> "tuple[mtdb, Dict[str, Any]]": + """Assign the resolved taxonomy dictionaries to this MTDB's taxonomy column""" + if forbid is None: + forbid = self._forbidden_tax_ranks + tax_dicts = self._reconcile_tax_dicts(set(self["genus"]), tax_dicts, forbid) + for i, genus in enumerate(self["genus"]): + self["taxonomy"][i] = tax_dicts[genus] + return mtdb(self), tax_dicts + + def infer_rank(self, lineage: str) -> str: + """Identify the taxonomic rank associated with an inputted lineage of + interest""" + linlow, rank = lineage.lower(), None + for ome, row in self.items(): + if linlow in set([x.lower() for x in row["taxonomy"].values()]): + rev_dict = { + k.lower(): v + for k, v in zip(row["taxonomy"].values(), row["taxonomy"].keys()) + } + rank = rev_dict[lineage] + + if not rank: + raise KeyError(f"no entry for {lineage}") + + return rank + + def extract_unique(self, allowed: int = 1, rank: str = "species") -> "mtdb": + """Extract unique rank from an MTDB""" + keys = copy.deepcopy(list(self.keys())) + random.shuffle(keys) + prep_db0 = {x: self[x] for x in keys} + prep_db1 = mtdb().set_index("ome") + if rank == "strain": + found = set() + for ome, row in prep_db0.items(): + name = row["taxonomy"]["species"] + " " + row["strain"] + if name not in found: + prep_db1[ome] = row + found_prep = list(found) + found_prep.append(name) + found = set(found_prep) + else: + found = defaultdict(int) + for ome, row in prep_db0.items(): + name = row["taxonomy"][rank] + found[name] += 1 + if found[name] <= allowed: + prep_db1[ome] = row + + return prep_db1 + + def extract_tax(self, lineages: Union[str, Iterable[str]]) -> "mtdb": + """Extract specific taxonomic lineages of interest based on their rank""" + if isinstance(lineages, str): + lineages = [lineages] + lineages = set(x.lower() for x in lineages) + rank_dict = {k: self.infer_rank(k) for k in list(lineages)} + ranks = list(set(rank_dict.values())) + + new_db = mtdb().set_index() + for ome in self: + for rank in ranks: + try: + if self[ome]["taxonomy"][rank].lower() in lineages: + new_db[ome] = self[ome] + except KeyError: # invalid rank key for row + pass # probably should standardize tax jsons period + + return new_db + + def extract_ome(self, omes: Iterable[str], column: str = "ome") -> "mtdb": + """Extract a list of genome codes (omes) of interest""" + new_db = mtdb().set_index(column) + db = self.set_index(column) + for i in db: + if i in list(omes): + new_db[i] = db[i] + return new_db.set_index() + + def extract_source(self, source: str) -> "mtdb": + """Extract an MTDB with genomes from a particular source""" + return mtdb( + { + ome: row + for ome, row in self.items() + if row["source"].lower() == source.lower() + }, + index="ome", + ) + + def extract_pub(self) -> "mtdb": + """Extract only published and usable genomes""" + new_db = mtdb().set_index() + for ome, row in self.items(): + if row["published"]: + new_db[ome] = row + return new_db + def getLogin(ncbi, jgi): @@ -368,7 +557,7 @@ def encrypt_pw( hash_pwd, hash_check = True, False while hash_pwd != hash_check: if hash_pwd != True: - eprint("ERROR: passwords do not match", flush=True) + logger.error("passwords do not match") hash_pwd = getpass.getpass(prompt="New MycotoolsDB login password: ") hash_check = getpass.getpass(prompt="Confirm password: ") @@ -383,7 +572,7 @@ def encrypt_pw( def loginCheck(info_path="~/.mycotools/mtdb_key", ncbi=True, jgi=True, encrypt=False): salt = b"D9\x82\xbfSibW(\xb1q\xeb\xd1\x84\x118" # NEED to make this store a password - if os.path.isfile(format_path(info_path)): + if Path(format_path(info_path)).is_file(): from cryptography.fernet import Fernet from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from cryptography.hazmat.backends import default_backend @@ -409,9 +598,7 @@ def loginCheck(info_path="~/.mycotools/mtdb_key", ncbi=True, jgi=True, encrypt=F decrypted = fernet.decrypt(data) data = decrypted.decode("UTF-8").split() if len(data) != 4: - eprint( - "BAD PASSWORD FILE. Delete ~/.mycotools/mtdb_key to reset.", flush=True - ) + logger.error("BAD PASSWORD FILE. Delete ~/.mycotools/mtdb_key to reset.") sys.exit(8) ncbi_email = data[0].rstrip() ncbi_api = data[1].rstrip() @@ -478,7 +665,7 @@ def primaryDB(path="$MYCODB", verbose=True): return None files = collect_files(full_path, "mtdb") - basenames = [os.path.basename(x) for x in files] + basenames = [Path(x).name for x in files] dates = [x.replace(".mtdb", "") for x in basenames if re.search(r"^\d+\.mtdb$", x)] primary = "19991231" # arbitrary primary for sorting for date in dates: @@ -488,7 +675,7 @@ def primaryDB(path="$MYCODB", verbose=True): primary = date if primary == "19991231": # if it is the arbitrary start if verbose: - eprint("\nWARNING: Primary MTDB not found in " + full_path, flush=True) + logger.warning("Primary MTDB not found in " + full_path) return None primary_path = format_path("$" + path + "/" + primary + ".mtdb") @@ -560,9 +747,9 @@ def df2db(df, db_path, header=False, overwrite=False, std_col=True, rescue=True) db_path = format_path(db_path) elif overwrite: number = 0 - while os.path.exists(db_path): + while Path(db_path).exists(): number += 1 - db_path = os.path.normpath(db_path) + "_" + str(number) + db_path = str(Path(db_path)) + "_" + str(number) if std_col: df = df2std(df) @@ -573,18 +760,15 @@ def df2db(df, db_path, header=False, overwrite=False, std_col=True, rescue=True) break except FileNotFoundError: if rescue: - eprint( - "\nOutput directory does not exist. Attempting to save in home folder.", - flush=True, + logger.warning( + "Output directory does not exist. Attempting to save in home folder." ) - db_path = "~/" + os.path.basename(os.path.normpath(db_path)) + db_path = "~/" + Path(str(Path(db_path))).name df.to_csv(db_path, sep="\t", index=None) raise FileNotFoundError break else: - eprint( - "\nOutput directory does not exist. Rescue not enabled.", flush=True - ) + logger.error("Output directory does not exist. Rescue not enabled.") raise FileNotFoundError break @@ -608,7 +792,7 @@ def hit2taxonomy( except urllib.error.HTTPError as goon: count += 1 if count == 5 and skip: - print("\n5 failed HTTP queries. Is NCBI down?", flush=True) + logger.error("5 failed HTTP queries. Is NCBI down?") sys.exit(100) if 500 <= goon.code <= 599: time.sleep(1) @@ -623,10 +807,10 @@ def hit2taxonomy( time.sleep(1) count += 1 if count == 5: - eprint("\nERROR: 5 failed taxonomy queries. " + str(taxid), flush=True) - eprint(tax_handle, flush=True) + logger.error("5 failed taxonomy queries. " + str(taxid)) + logger.debug("%s", tax_handle) for line in tax_handle: - print(line, flush=True) + logger.debug(line) if skip: sys.exit(1) records = False @@ -635,9 +819,9 @@ def hit2taxonomy( if "latin-1" in str(tax_handle): count = 0 - eprint("\tERROR: latin-1 encoding", flush=True) + logger.error("latin-1 encoding") for line in tax_handle: - print(line, flush=True) + logger.debug(line) time.sleep(30) Entrez.email = email Entrez.api_key = api @@ -656,40 +840,34 @@ def hit2taxonomy( return tax_dict, sleep -def prepare_tax_dicts(df, tax_dicts={}): - """Identify the genera that do not have higher taxonomy ascribed to them""" +def prepare_tax_dicts(df, tax_dicts=None): + """Identify the genera that do not have higher taxonomy ascribed to them. - need_tax = set() + Backwards-compatible dispatcher: MTDBs (and MTDB-shaped dicts) use + ``mtdb.prepare_tax_dicts``; a deprecated pandas DataFrame uses the path + below.""" + if tax_dicts is None: + tax_dicts = {} # is the df of mtdb class or have a taxonomy column as a list? - if isinstance(df, mtdb) or isinstance(df["taxonomy"], list): - df = df.set_index("ome") - for k, v in df.items(): - if v["genus"] in tax_dicts: - continue - tax_json = read_tax(v["taxonomy"]) - if any( - v - for k, v in tax_json.items() - if k not in {"genus", "species", "strain"} - ): - tax_dicts[v["genus"]] = tax_json - else: - need_tax.add(v["genus"]) + if isinstance(df, mtdb): + return df.prepare_tax_dicts(tax_dicts) + if isinstance(df["taxonomy"], list): + return mtdb(df).prepare_tax_dicts(tax_dicts) # otherwise it is a pandas dataframe - else: - df["taxonomy"] = df["taxonomy"].fillna({}) - for k, v in df.iterrows(): - if v["genus"] in tax_dicts: - continue - tax_json = read_tax(v["taxonomy"]) - if any( - v - for k, v in tax_json.items() - if k not in {"genus", "species", "strain"} - ): - tax_dicts[v["genus"]] = tax_json - else: - need_tax.add(v["genus"]) + need_tax = set() + df["taxonomy"] = df["taxonomy"].fillna({}) + for k, v in df.iterrows(): + if v["genus"] in tax_dicts: + continue + tax_json = read_tax(v["taxonomy"]) + if any( + name + for rank, name in tax_json.items() + if rank not in {"genus", "species", "strain"} + ): + tax_dicts[v["genus"]] = tax_json + else: + need_tax.add(v["genus"]) need_tax = set(need_tax).difference(set(tax_dicts.keys())) return need_tax, tax_dicts @@ -712,7 +890,7 @@ def query_ncbi4taxonomy(genus, api_key, king, rank, count=0): count += 1 if not ids: - eprint(f"\t\t{genus} TaxID acquisition failed", flush=True) + logger.warning(f"{genus} TaxID acquisition failed") return None, count # for each taxID acquired, fetch the actual taxonomy information @@ -752,14 +930,14 @@ def query_ncbi4taxonomy(genus, api_key, king, rank, count=0): if lineage["ScientificName"].lower() == king.lower(): taxid = tax if len(ids) > 1: - print("\t\tMultiple Tax IDs: " + str(ids), flush=True) + logger.warning("Multiple Tax IDs: " + str(ids)) break else: for lineage in lineages: taxid = tax if taxid == 0: - eprint(f"\t\t{genus} not recovered in {king}", flush=True) + logger.warning(f"{genus} not recovered in {king}") return None, count # for each taxonomic classification, add it to the taxonomy dictionary string @@ -797,7 +975,7 @@ def gather_taxonomy( with open(output_path + ".tmp", "w") as out: for genus, tax_dict in tax_dicts.items(): out.write(f"{genus}\t{json.dumps(tax_dict)}\n") - os.rename(f"{output_path}.tmp", output_path) + Path(f"{output_path}.tmp").rename(output_path) return tax_dicts @@ -824,7 +1002,7 @@ def gather_taxonomy_dataset( with open(tax_accs_file, "w") as out: out.write("\n".join(sorted(need_tax))) - cwd = os.getcwd() + cwd = str(Path.cwd()) os.chdir(output_path) cmd_scaf = [ "datasets", @@ -852,12 +1030,12 @@ def gather_taxonomy_dataset( except zipfile.BadZipFile: count += 1 if count == 3: - eprint("ERROR: taxonomy acquisition failed", flush=True) + logger.error("taxonomy acquisition failed") sys.exit(130) continue break - os.remove(dataset_path) + Path(dataset_path).unlink() unzip_path = output_path + "ncbi_dataset/" tax_dicts = parse_dataset_taxonomy( @@ -901,81 +1079,33 @@ def parse_dataset_taxonomy(tax_json, tax_dicts, tax_head, rank_head): return tax_dicts -def read_tax(taxonomy_string): - """Read taxonomy from an MTDB by converting the string into a dictionary""" - tax_strs = [ - "superkingdom", - "kingdom", - "phylum", - "subphylum", - "class", - "order", - "family", - "subfamily", - ] - if taxonomy_string: - if isinstance(taxonomy_string, str): - dict_string = taxonomy_string.replace("'", '"') - try: - tax_dict = json.loads(dict_string) - except TypeError: - tax_dict = {} - else: - tax_dict = taxonomy_string - try: - tax_dict = {**tax_dict, **{x: "" for x in tax_strs if x not in tax_dict}} - except TypeError: # inappropriate tax_dict in the column - tax_dict = {x: "" for x in tax_strs} - return tax_dict - else: - return {} +# read_tax is defined as mtdb.read_tax; expose it at module scope for the +# historical ``from mycotools.lib.dbtools import read_tax`` import. +read_tax = mtdb.read_tax # assimilate taxonomy dictionary strings and append the resulting taxonomy string dicts to an inputted database # forbid a list of taxonomic classifications you are not interested in and return a new database -def assimilate_tax( - db, - tax_dicts, - ome_index="ome", - forbid={ - "no rank", - "subkingdom", - "genus", - "species", - "species group", - "varietas", - "forma", - }, -): - - genera = set(db["genus"]) - tax_dicts = {x: tax_dicts[x] for x in tax_dicts if tax_dicts[x]} - for genus in tax_dicts: - tax_dicts[genus] = { - x: tax_dicts[genus][x] for x in tax_dicts[genus] if x not in forbid - } - missing = list(genera.difference(set(tax_dicts.keys()))) - - for miss in missing: - tax_dicts[miss] = {} +def assimilate_tax(db, tax_dicts, ome_index="ome", forbid=None): + """Backwards-compatible dispatcher for ``mtdb.assimilate_tax``; retains the + deprecated pandas DataFrame path.""" + if forbid is None: + forbid = mtdb._forbidden_tax_ranks if isinstance(db, mtdb): - for i, genus in enumerate(db["genus"]): - db["taxonomy"][i] = tax_dicts[genus] - return mtdb(db), tax_dicts - else: - for i, row in db.iterrows(): - db.at[i, "taxonomy"] = tax_dicts[row["genus"]] - + return db.assimilate_tax(tax_dicts, forbid=forbid) + tax_dicts = mtdb._reconcile_tax_dicts(set(db["genus"]), tax_dicts, forbid) + for i, row in db.iterrows(): + db.at[i, "taxonomy"] = tax_dicts[row["genus"]] return db, tax_dicts def parse_user_config(mtdb_config_file=format_path("~/.mycotools/config.json")): config_dir = format_path("~/.mycotools/") - if not os.path.isdir(config_dir): - os.mkdir(config_dir) + if not Path(config_dir).is_dir(): + Path(config_dir).mkdir() config_dir += "/" - if os.path.isfile(mtdb_config_file): + if Path(mtdb_config_file).is_file(): config = read_json(mtdb_config_file) else: config = {"log": {}} @@ -1008,9 +1138,9 @@ def mtdb_initialize( mtdb_config = read_json(mycodb_loc + "config/mtdb.json") dbtype = mtdb_config["branch"] - eprint("Establishing " + dbtype + " connection", flush=True) + logger.info("Establishing " + dbtype + " connection") - if not os.path.isdir(mycodb_loc + "mtdb/") and not init: + if not Path(mycodb_loc + "mtdb/").is_dir() and not init: raise FileNotFoundError("invalid MycotoolsDB path") dPath = mycodb_loc + "data/" user_config[mycodb_loc] = { @@ -1031,7 +1161,7 @@ def mtdb_initialize( interface = format_path("~/.mycotools/config.json") -if os.path.isfile(interface): +if Path(interface).is_file(): envs_info = read_json(interface) if envs_info["active"]: for var, env in envs_info[envs_info["active"]].items(): diff --git a/mycotools/lib/kontools.py b/mycotools/lib/kontools.py index 4421b3e..84f9d91 100755 --- a/mycotools/lib/kontools.py +++ b/mycotools/lib/kontools.py @@ -6,14 +6,63 @@ import glob import gzip import json +import logging import shutil import tarfile import argparse import subprocess +from pathlib import Path +from contextlib import contextmanager from tqdm import tqdm from datetime import datetime +logger = logging.getLogger(__name__) + + +class _LevelFormatter(logging.Formatter): + """Prefix records with their level name, except INFO, which is emitted + verbatim so status messages read as they did before the logging + migration.""" + + _PREFIX = { + logging.DEBUG: "DEBUG: ", + logging.INFO: "", + logging.WARNING: "WARNING: ", + logging.ERROR: "ERROR: ", + logging.CRITICAL: "CRITICAL: ", + } + + def format(self, record): + message = super().format(record) + prefix = self._PREFIX.get(record.levelno, "") + # Do not double-prefix messages that already carry their level word. + if prefix and message.lstrip().upper().startswith(prefix.strip().rstrip(":")): + prefix = "" + return prefix + message + + +def setup_logging(verbose=False, level=None): + """Configure logging for the Mycotools CLI. + + Attaches a single stderr handler to the root logger (idempotent across + repeated calls) and sets the ``mycotools`` logger to INFO, or DEBUG when + ``verbose`` is True. The root logger stays at WARNING so third-party + libraries do not clutter diagnostic output. Call once from a script's + entry point after parsing arguments.""" + if level is None: + level = logging.DEBUG if verbose else logging.INFO + root = logging.getLogger() + root.setLevel(logging.WARNING) + if not any(getattr(h, "_mycotools", False) for h in root.handlers): + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter(_LevelFormatter("%(message)s")) + handler._mycotools = True + root.addHandler(handler) + logging.getLogger("mycotools").setLevel(level) + return logging.getLogger("mycotools") + + class kon_log: """A print class designed to enable swift string formatting while moving between scripts""" @@ -127,7 +176,7 @@ def parse_run_log(log_path, args_dict, fail=set()): if isinstance(args_dict, argparse.Namespace): args_dict = namespace_to_dict(args_dict) - if not os.path.isfile(log_path): + if not Path(log_path).is_file(): write_json(args_dict, log_path) return {} else: @@ -286,28 +335,28 @@ def getColors(size, ignore=[], rgb=False): def tardir(dir_, rm=True): - if not os.path.isdir(format_path(dir_)): + if not Path(format_path(dir_)).is_dir(): return False with tarfile.open(format_path(dir_)[:-1] + ".tar.gz", "w:gz") as tar: - tar.add(dir_, arcname=os.path.basename(format_path(dir_)[:-1])) + tar.add(dir_, arcname=Path(format_path(dir_)[:-1]).name) if rm: shutil.rmtree(dir_) def untardir(dir_, rm=False, to=None): if not to: - to = os.path.dirname(dir_[:-1]) + to = str(Path(dir_[:-1]).parent) tar = tarfile.TarFile.open(dir_) tar.extractall(path=to) tar.close() if rm: - os.remove(dir_) + Path(dir_).unlink() def checkdir(dir_, unzip=False, to=None, rm=False): - if os.path.isdir(dir_): + if Path(dir_).is_dir(): return True - elif os.path.isfile(format_path(dir_) + ".tar.gz"): + elif Path(format_path(dir_) + ".tar.gz").is_file(): if unzip: if dir_.endswith("/"): dir_ = dir_[:-1] @@ -316,31 +365,6 @@ def checkdir(dir_, unzip=False, to=None, rm=False): return False -def eprint(*args, **kwargs): - """Prints to stderr""" - print(*args, file=sys.stderr, **kwargs) - - -def fprint(out_str, log): - with open(log, "a") as out: - out.write(args) - - -def zprint(out_str, log=None, flush=True): - fprint(out_str, log) - print(out_str, flush=flush) - - -def vprint(toPrint, v=False, e=False, flush=True): - """Boolean print option to stdout or stderr (e)""" - - if v: - if e: - eprint(toPrint, flush=True) - else: - print(toPrint, flush=True) - - def read_json(config_path, compress=False): if compress or config_path.endswith(".gz"): @@ -362,6 +386,32 @@ def write_json(obj, json_path, compress=False, indent=1, **kwargs): json.dump(obj, json_out, indent=indent, **kwargs) +@contextmanager +def atomic_write(path, mode="w", suffix=".tmp", **kwargs): + """Context manager for crash-safe file writes. + + Yields a handle to a temporary sibling file (`path` + `suffix`) and, only + upon leaving the block without error, atomically replaces `path` with it. + Because the temporary file lives beside the target it shares a filesystem, + so the final replace is atomic and can never leave a half-written `path` + behind. If the block raises, the temporary file is removed and `path` is + left untouched. + + Consolidates the `open(path + '.tmp')` ... `shutil.move`/`Path.rename` + idiom repeated throughout the codebase. + """ + path = Path(path) + tmp_path = path.with_name(path.name + suffix) + try: + with open(tmp_path, mode, **kwargs) as handle: + yield handle + tmp_path.replace(path) + except BaseException: + if tmp_path.is_file(): + tmp_path.unlink() + raise + + def gunzip(gzip_file, remove=True, spacer="\t"): """gunzips gzip_file and removes if successful""" @@ -372,12 +422,12 @@ def gunzip(gzip_file, remove=True, spacer="\t"): for line in f_in: f_out.write(line) if remove: - os.remove(gzip_file) + Path(gzip_file).unlink() return new_file except: - if os.path.isfile(new_file): - if os.path.isfile(gzip_file): - os.remove(new_file) + if Path(new_file).is_file(): + if Path(gzip_file).is_file(): + Path(new_file).unlink() raise IOError("gunzip " + str(gzip_file) + " failed") @@ -473,23 +523,20 @@ def findExecs(deps, exit=set(), verbose=True): then exit. """ - vprint("\nDependency check:", v=verbose, e=True, flush=True) + logger.debug("Dependency check:") checks, failed = [], [] if isinstance(deps, str): deps = [deps] for dep in sorted(deps): check = shutil.which(dep) - vprint("{:<15}".format(dep + ":", flush=True) + str(check), v=verbose, e=True) + logger.debug("{:<15}".format(dep + ":") + str(check)) if not check and dep in exit: failed.append(dep) else: checks.append(check) if failed: - eprint("\nERROR: missing dependencies:", flush=True) - for f in failed: - vprint(f, v=verbose, e=True) - eprint() + logger.error("missing dependencies: " + ", ".join(failed)) sys.exit(300) return checks @@ -501,61 +548,45 @@ def findEnvs(envs, exit=set(), verbose=True): If env is not in path and it is in exit, exit. """ - vprint("\nEnvironment check:", v=verbose, e=True, flush=True) + logger.debug("Environment check:") if type(envs) is str: envs = [envs] - eprint(flush=True) for env in envs: try: - vprint( - "{:<15}".format(env + ":", flush=True) + str(os.environ[env]), - v=verbose, - e=True, - ) + logger.debug("{:<15}".format(env + ":") + str(os.environ[env])) except KeyError: - vprint("{:<15}".format(env + ":", flush=True) + "None", v=verbose, e=True) + logger.debug("{:<15}".format(env + ":") + "None") if env in exit: - eprint("\nERROR: " + env + " not in PATH", flush=True) + logger.error(env + " not in PATH") sys.exit(301) -def expandEnvVar(path): - """Expands environment variables by regex substitution""" - - envs = re.findall(r"\$[^/]+", path) - for env in envs: - path = path.replace(env, os.environ[env.replace("$", "")]) - - return path.replace("//", "/") - - def format_path(path, force_dir=False): - """Goal is to convert all path types to absolute path with explicit dirs""" + """Convert a path to an absolute path with explicit directories. + + Expands ``~`` and ``$VAR`` (raising KeyError on an undefined variable). By + convention an existing directory is returned with a trailing ``/`` while + files and non-existent paths have none; ``force_dir`` forces a trailing + ``/`` for directories that do not exist yet. Symlinks are not resolved.""" - # path = path.replace('//','/') - # except AttributeError: # not a string - # return None # removed this because let it be handled on the other end - # try: if path: - path = os.path.expanduser(path) - path = expandEnvVar(path) - # path = os.path.abspath( path ) - # except TypeError: - # return None again, let this be handled on the other end to increase - # throughput + path = str(Path(str(path)).expanduser()) + for env in re.findall(r"\$[^/]+", path): + path = path.replace(env, os.environ[env.replace("$", "")]) + path = path.replace("//", "/") if force_dir: if not path.endswith("/"): path += "/" else: if path.endswith("/"): - if not os.path.isdir(path): + if not Path(path).is_dir(): path = path[:-1] else: - if os.path.isdir(path): + if Path(path).is_dir(): path += "/" if not path.startswith("/"): - path = os.getcwd() + "/" + path + path = str(Path.cwd()) + "/" + path path = path.replace("/./", "/") while "/../" in path: @@ -580,17 +611,12 @@ def collect_files(directory="./", filetype="*", recursive=False): else: filetypes = [filetype] - directory = format_path(directory) + directory = Path(format_path(directory)) filelist = [] for filetype in filetypes: - if recursive: - filelist.extend( - glob.glob(directory + "/**/*." + filetype, recursive=recursive) - ) - else: - filelist.extend( - glob.glob(directory + "/*." + filetype, recursive=recursive) - ) + pattern = "*." + filetype + matches = directory.rglob(pattern) if recursive else directory.glob(pattern) + filelist.extend(str(match) for match in matches) return filelist @@ -603,7 +629,7 @@ def collect_dirs(input_glob, recursive=False): """ in_dirs = glob.glob(input_glob, recursive=recursive) - return [dir_ for dir_ in in_dirs if os.path.isdir(dir_)] + return [dir_ for dir_ in in_dirs if Path(dir_).is_dir()] def dictSplit(Dict, factor): @@ -647,13 +673,13 @@ def sys_start(args, usage, min_len, dirs=[], files=[]): elif len(args) < min_len: print("\n" + usage + "\n", flush=True) sys.exit(1) - elif not all(os.path.isfile(format_path(x)) for x in files): + elif not all(Path(format_path(x)).is_file() for x in files): print("\n" + usage, flush=True) - eprint("ERROR: input file(s) do not exist\n", flush=True) + logger.error("input file(s) do not exist") sys.exit(3) - elif not all(os.path.isfile(format_path(x)) for x in dirs): + elif not all(Path(format_path(x)).is_file() for x in dirs): print("\n" + usage, flush=True) - eprint("ERROR: input directory does not exist\n", flush=True) + logger.error("input directory does not exist") sys.exit(4) return args @@ -709,12 +735,7 @@ def intro(script_name, args_dict, credit="", log=False, stdout=True): for arg in args_dict: out_str += "\n" + "{:<30}".format(arg.upper() + ":") + str(args_dict[arg]) - if log: - zprint(out_str, log) - elif stdout: - print(out_str, flush=True) - else: - eprint(out_str, flush=True) + logger.info(out_str) return start_time @@ -736,12 +757,7 @@ def outro(start_time, log=False, stdout=True): + " minutes\n" ) - if log: - zprint(out_str, log) - elif not stdout: - eprint(out_str, flush=True) - else: - print(out_str, flush=True) + logger.info(out_str) sys.exit(0) @@ -754,17 +770,17 @@ def prep_output(output, mkdir=True, require_newdir=False, cd=False): """ output = format_path(output, force_dir=True) - if os.path.isdir(output): + if Path(output).is_dir(): if require_newdir: - eprint("\nERROR: directory exists.", flush=True) + logger.error("directory exists.") return None - elif os.path.exists(output): - output = os.path.dirname(output) + elif Path(output).exists(): + output = str(Path(output).parent) else: if not mkdir: - eprint("\nERROR: directory does not exist.", flush=True) + logger.error("directory does not exist.") return None - os.mkdir(output) + Path(output).mkdir() if cd: os.chdir(output) @@ -774,8 +790,8 @@ def prep_output(output, mkdir=True, require_newdir=False, cd=False): def mkOutput(base_dir, program, reuse=True, suffix=datetime.now().strftime("%Y%m%d")): if not base_dir: - base_dir = os.getcwd() + "/" - if not os.path.isdir(format_path(base_dir)): + base_dir = str(Path.cwd()) + "/" + if not Path(format_path(base_dir)).is_dir(): raise FileNotFoundError(base_dir + " does not exist") if suffix: out_dir = format_path(base_dir) + program + "_" + suffix @@ -784,14 +800,14 @@ def mkOutput(base_dir, program, reuse=True, suffix=datetime.now().strftime("%Y%m if not reuse: count, count_dir = 1, out_dir - while os.path.isdir(count_dir): + while Path(count_dir).is_dir(): count_dir += "_" + str(count) count += 1 - os.mkdir(count_dir) + Path(count_dir).mkdir() return count_dir + "/" else: - if not os.path.isdir(out_dir): - os.mkdir(out_dir) + if not Path(out_dir).is_dir(): + Path(out_dir).mkdir() return out_dir + "/" @@ -809,12 +825,12 @@ def checkDep(dep_list=[], var_list=[], exempt=set()): check.append(False) failedVars.append(var) if not all(check): - eprint("\nERROR: Dependencies not met:", flush=True) + logger.error("Dependencies not met:") for dep in dep_list: if not shutil.which(dep): - eprint(dep + " not in PATH", flush=True) + logger.error(dep + " not in PATH") for failed in failedVars: - eprint(failed + " variable not set", flush=True) + logger.error(failed + " variable not set") sys.exit(135) diff --git a/mycotools/manage_mtdb.py b/mycotools/manage_mtdb.py index 8cb36aa..1623178 100755 --- a/mycotools/manage_mtdb.py +++ b/mycotools/manage_mtdb.py @@ -2,9 +2,13 @@ import os import sys +import logging import argparse from mycotools.lib.dbtools import loginCheck, primaryDB, mtdb, encrypt_pw -from mycotools.lib.kontools import format_path, read_json, collect_files +from mycotools.lib.kontools import format_path, read_json, collect_files, setup_logging +from pathlib import Path + +logger = logging.getLogger(__name__) def rm_outdated(omes, yes=False): @@ -13,19 +17,19 @@ def rm_outdated(omes, yes=False): biofiles, to_del = [], [] # compile the files biofiles.extend( - [f"{os.environ['MYCOGFF3']}/{x}" for x in os.listdir(os.environ["MYCOGFF3"])] + [f"{os.environ['MYCOGFF3']}/{x}" for x in [p.name for p in Path(os.environ["MYCOGFF3"]).iterdir()]] ) biofiles.extend( - [f"{os.environ['MYCOFAA']}/{x}" for x in os.listdir(os.environ["MYCOFAA"])] + [f"{os.environ['MYCOFAA']}/{x}" for x in [p.name for p in Path(os.environ["MYCOFAA"]).iterdir()]] ) biofiles.extend( - [f"{os.environ['MYCOFNA']}/{x}" for x in os.listdir(os.environ["MYCOFNA"])] + [f"{os.environ['MYCOFNA']}/{x}" for x in [p.name for p in Path(os.environ["MYCOFNA"]).iterdir()]] ) # remove each biofile for i in biofiles: ome = None - ome_prep = os.path.basename(i) + ome_prep = Path(i).name if ome_prep.endswith(".gff3"): ome = ome_prep[:-5] elif ome_prep.endswith(".faa"): @@ -44,7 +48,7 @@ def rm_outdated(omes, yes=False): data = input(f"\n{len(to_del)} omes to be deleted.\n" + "Continue [y/N]? ") if data.lower() in {"yes", "y"}: for i in to_del: - os.remove(i) + Path(i).unlink() else: raise KeyError("cache removal stopped") @@ -65,7 +69,7 @@ def restrictions( for r, s, reason in restr_list: if s.lower() in {"ncbi", "jgi"} and r not in accs: restricted.append([r, s.lower(), str(reason)]) - print(r, s, flush=True) + logger.info("%s %s", r, s) in_db = [x[0] for x in restricted if x[0] in db] while in_db: @@ -101,6 +105,7 @@ def cli(): ) parser.add_argument("-y", "--yes", help="Answer yes", action="store_true") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) db = mtdb(primaryDB()).set_index("assembly_acc") diff --git a/mycotools/mtdb.py b/mycotools/mtdb.py index c4f679e..5fc31f3 100755 --- a/mycotools/mtdb.py +++ b/mycotools/mtdb.py @@ -9,199 +9,172 @@ # NEED to add option to export NCBI/JGI credentials # NEED to pay attention to old ome versions -import os import re import sys +import logging +import argparse import subprocess -from mycotools.lib.kontools import format_path, read_json, write_json, eprint +from pathlib import Path +from mycotools.lib.kontools import format_path, setup_logging from mycotools.lib.dbtools import ( primaryDB, - mtdb_connect, mtdb_disconnect, mtdb_initialize, mtdb, - loginCheck, parse_user_config, ) +logger = logging.getLogger(__name__) -def get_version(): - from importlib.metadata import version +# subcommand name/alias -> delegated standalone console script +SUBCOMMANDS = { + "extract": "extract_mtdb", + "e": "extract_mtdb", + "update": "update_mtdb", + "u": "update_mtdb", + "predb2mtdb": "predb2mtdb", + "p": "predb2mtdb", + "manage": "manage_mtdb", + "m": "manage_mtdb", +} - print(f'Mycotools version {version("mycotools")}') +DESCRIPTION = """MycotoolsDB (MTDB) utility +Run without arguments to print the primary MTDB path. -def main(argv=sys.argv): - description = ( - "MycotoolsDB (MTDB) utility usage" - + "\n\nmtdb: print master MTDB path" - + "\n\nmtdb :" - + "\n[e]xtract\t\textract sub .mtdb file" - + "\n[u]pdate\t\tupdate/initialize primary MTDB" - + "\n[p]redb2mtdb\t\tadd local genomes to the primary MTDB" - + "\n[m]anage\t\tMTDB management utility" - + "\n\nmtdb [.gff3|.fna|.faa]: [PATH] print ome/ome code path" - + "\n\nmtdb :" - + "\n[-i DBPATH]\t[--interface]\tInitialize MTDB linkage" - + "\n[-u]\t\t[--unlink]\tUnlink from MTDB" - + "\n[-l]\t\t[--list]\tList historically linked primary MTDBs" - + "\n[-d]\t\t[--dependency]\tInstall/update dependencies" - + "\n[-v]\t\t[--version]\tPrint Mycotools version and exit" - ) - # + '\n[-f]\t\t[--fungi] \tConnect to fungal MTDB' \ - # + '\n[-p]\t\t[--prokaryote]\tConnect to prokaryote MTDB' \ +Subcommands (all following arguments are forwarded to the standalone tool): + extract (e) extract a sub-.mtdb file + update (u) update / initialize the primary MTDB + predb2mtdb (p) add local genomes to the primary MTDB + manage (m) MTDB management utility - config = parse_user_config() - if any([not x.startswith("-") for x in argv[1:]]): - script = [x for x in argv[1:] if not x.startswith("-")] - script = script[0] - ab2mt = { - "extract": "extract_mtdb", - "update": "update_mtdb", - "predb2mtdb": "predb2mtdb", - "e": "extract_mtdb", - "u": "update_mtdb", - "p": "predb2mtdb", - "m": "manage_mtdb", - "manage": "manage_mtdb", - } - if script in ab2mt: - exit_code = subprocess.call( - [ab2mt[script]] + [x for x in argv[1:] if x != script] - ) - sys.exit(exit_code) - - if len([x for x in argv if x.startswith("-")]) > 1: - eprint("\nERROR: one argument allowed.\n" + description, flush=True) - sys.exit(2) - - set_argv = set(argv) - if {"-h", "--help"}.intersection(set_argv): - print("\n" + description + "\n", flush=True) - sys.exit(0) - elif {"-v", "--version"}.intersection(set_argv): - get_version() - sys.exit(0) +Ome lookup: + mtdb [.gff3|.fna|.faa] print an ome's row, or a specific file path""" - if {"-i", "--interface"}.intersection(set_argv): - if "-i" in set_argv: - coord = "-i" - else: - coord = "--interface" - try: - mycodb_loc_prep = argv[argv.index(coord) + 1] - if isinstance(mycodb_loc_prep, str): - if mycodb_loc_prep.startswith("-"): - raise IndexError - else: - mycodb_loc = format_path(mycodb_loc_prep) - if not os.path.isdir(mycodb_loc): - raise FileNotFoundError("invalid MTDB path") - elif not os.path.isdir(mycodb_loc + "mtdb"): - raise FileNotFoundError("invalid MTDB path") - except IndexError: - raise ValueError("--interface requires a path") - mtdb_initialize(mycodb_loc) - elif {"-d", "--dependencies"}.intersection(set_argv): - pip_deps = ["dna_features_viewer", "mycotools"] - dep_cmds = [ - ["conda", "install", "-y", "clipkit"], - ["python3", "-m", "pip", "install"] + pip_deps + ["--upgrade"], - ] - for dep_cmd in dep_cmds: - cmd = subprocess.call(dep_cmd) - if cmd: - print("\nUPDATE failed. Exit " + str(dep_cmd)) - sys.exit(cmd) - elif {"-l", "--list"}.intersection(set_argv): - if "log" in config: - for dbtype in config["log"]: - print(dbtype, flush=True) - for mtdb_loc, login_time in config["log"][dbtype].items(): - print(f"\t{mtdb_loc} {login_time}", flush=True) - print() - sys.exit(0) - # elif {'-f', '--fungi'}.intersection(set_argv): - # if len(set_argv) > 2: - # eprint('\nERROR: -f does not accept additional arguments. Did you mean -i?') - # sys.exit(14) - # if '-f' in set_argv: - # coord = '-f' - # else: - # coord = '--fungi' - # if 'fungi' not in config: - # raise ValueError('fungal MTDB not connected') - # else: - # mtdb_connect(config, 'fungi') - # if not os.path.isfile(format_path('~/.mycotools/mtdb_key')): - # loginCheck() - # elif {'-p', '--prokaryote'}.intersection(set_argv): - # if len(set_argv) > 2: - # eprint('\nERROR: -p does not accept additional arguments. Did you mean -i?') - # sys.exit(15) - # - # if '-p' in set_argv: - # coord = '-p' - # else: - # coord = '--prokaryote' - # if 'prokaryote' not in config: - # raise ValueError('prokaryote MTDB not connected') - # else: - # mtdb_connect(config, 'prokaryote') - # if not os.path.isfile(format_path('~/.mycotools/mtdb_key')): - # loginCheck() - elif {"-u", "--unlink"}.intersection(set_argv): - if "-u" in set_argv: - coord = "-u" +def get_version(): + """Return the `-v`/`--version` output string.""" + from importlib.metadata import version + + return f'Mycotools version {version("mycotools")}' + + +def build_parser(): + """Build the base `mtdb` argument parser. + + Subcommands and ome-lookup share one positional (`target`) followed by a + REMAINDER: this lets subcommand arguments pass through verbatim to their + standalone tool (native argparse subparsers cannot, as they intercept + forwarded flags and reject arbitrary ome positionals).""" + parser = argparse.ArgumentParser( + prog="mtdb", + description=DESCRIPTION, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("-v", "--version", action="version", version=get_version()) + ops = parser.add_mutually_exclusive_group() + ops.add_argument( + "-i", "--interface", metavar="DBPATH", help="link/initialize the MTDB at path" + ) + ops.add_argument( + "-u", "--unlink", action="store_true", help="unlink from the MTDB" + ) + ops.add_argument( + "-l", + "--list", + action="store_true", + help="list historically linked primary MTDBs", + ) + parser.add_argument( + "target", + nargs="?", + metavar="SUBCOMMAND|OME", + help="subcommand (see below) or ome code to look up", + ) + parser.add_argument("rest", nargs=argparse.REMAINDER, help=argparse.SUPPRESS) + return parser + + +def delegate(script, args): + """Forward a subcommand to its standalone console script; return its exit + code.""" + return subprocess.call([script] + args) + + +def link_mtdb(path): + """Link (interface) to the MTDB rooted at `path`.""" + mtdb_loc = format_path(path) + if not Path(mtdb_loc).is_dir() or not Path(mtdb_loc + "mtdb").is_dir(): + raise FileNotFoundError("invalid MTDB path") + mtdb_initialize(mtdb_loc) + + +def list_links(config): + """Print the historically linked primary MTDBs recorded in the user config.""" + for dbtype in config.get("log", {}): + print(dbtype, flush=True) + for mtdb_loc, login_time in config["log"][dbtype].items(): + print(f"\t{mtdb_loc} {login_time}", flush=True) + print() + + +def lookup_omes(omes): + """Print the database row, or a specific file path, for ome code(s).""" + db = mtdb(primaryDB()).set_index() + for ome_prep in omes: + if ome_prep in db: + print(ome_prep + "\t" + "\t".join(str(v) for v in db[ome_prep].values())) + return + ome = re.sub(r"\.\w+[\w\d]$", "", ome_prep) + ext_srch = re.search(r"^\d+\.?\d*\.(.*$)", ome_prep[6:]) + extension = ext_srch[1] if ext_srch is not None else None + if ome in db: + try: + print(db[ome][extension] if extension else {"ome": ome, **db[ome]}) + except KeyError: + raise KeyError("Invalid extension " + extension) else: - coord = "--unlink" - mtdb_disconnect() - sys.exit(0) - elif len(argv) > 1: - omes = argv[1].replace('"', "").replace("'", "").split() - for ome_prep in omes: - db = mtdb(primaryDB()).set_index() - if ome_prep in db: - print( - ome_prep - + "\t" - + "\t".join([str(db[ome_prep][x]) for x in db[ome_prep]]) - ) - sys.exit(0) - ome = re.sub(r"\.\w+[\w\d]$", "", ome_prep) - extension_srch = re.search(r"^\d+\.?\d*\.(.*$)", ome_prep[6:]) - if extension_srch is not None: - extension = extension_srch[1] - else: - extension = None - if ome in db: - try: - if extension: - print(db[ome][extension]) - else: - print({**{"ome": ome}, **db[ome]}) - except KeyError: - raise KeyError("Invalid extension " + extension) + for ref_ome, row in db.items(): + if ref_ome.startswith(ome + "."): + print(row[extension] if extension else {"ome": ome, **row}) + break else: - for ref_ome, row in db.items(): - if ref_ome.startswith(ome + "."): - if extension: - print(row[extension]) - else: - print({**{"ome": ome}, **row}) - break - else: - raise KeyError("Invalid ome " + ome) - sys.exit(0) + raise KeyError("Invalid ome " + ome) + +def print_primary(): + """Print the primary MTDB path; return an exit code.""" path = primaryDB() if path: print(path, flush=True) + return 0 + logger.error("Link a MycotoolsDB via `mtdb -i `") + return 1 + + +def main(argv=sys.argv): + setup_logging() + config = parse_user_config() + args = build_parser().parse_args(argv[1:]) + + # 1. forward recognized subcommands to their standalone tool + if args.target in SUBCOMMANDS: + sys.exit(delegate(SUBCOMMANDS[args.target], args.rest)) + # 2. terminal base operations + if args.unlink: + mtdb_disconnect() + sys.exit(0) + if args.list: + list_links(config) + sys.exit(0) + # 3. ome-code lookup (skipped when linking, which reports the path afterward) + if args.target and not args.interface: + lookup_omes(args.target.replace('"', "").replace("'", "").split()) sys.exit(0) - else: - eprint("Link a MycotoolsDB via `mtdb -i `") - sys.exit(1) + # 4. optionally (re)link, then always report the primary MTDB path + if args.interface: + link_mtdb(args.interface) + sys.exit(print_primary()) def cli(): diff --git a/mycotools/ncbiAcc2fa.py b/mycotools/ncbiAcc2fa.py index 22ad287..24066a0 100755 --- a/mycotools/ncbiAcc2fa.py +++ b/mycotools/ncbiAcc2fa.py @@ -1,11 +1,14 @@ #! /usr/bin/env python3 -import os import sys import time +import logging import getpass from Bio import Entrez -from mycotools.lib.kontools import file2list, eprint, sys_start +from mycotools.lib.kontools import file2list, sys_start, setup_logging +from pathlib import Path + +logger = logging.getLogger(__name__) def entrez_login(): @@ -19,7 +22,7 @@ def entrez_login(): Entrez.api_key = api limit = 10 - eprint(flush=True) + print(flush=True) return limit @@ -32,7 +35,7 @@ def grab_accs(accs, limit): if count >= limit: count = 0 time.sleep(1) - eprint(acc, flush=True) + logger.info(acc) # iteratively query until successful attempt = 0 @@ -62,11 +65,12 @@ def cli(): ) # parse the arguments + setup_logging() args = sys_start(sys.argv, usage, 2) if len(args) <= 3: # import a file of accessions - if os.path.isfile(args[1]): + if Path(args[1]).is_file(): if len(args) == 3: accs = file2list(args[1], sep="\t", col=args[2]) else: @@ -78,7 +82,7 @@ def cli(): limit = entrez_login() out_str = grab_accs(accs, limit) - eprint(flush=True) + print(flush=True) with open(args + ".retr.fa", "w") as out: out.write(out_str) diff --git a/mycotools/ncbiDwnld.py b/mycotools/ncbiDwnld.py index 826f2d3..9bebbeb 100755 --- a/mycotools/ncbiDwnld.py +++ b/mycotools/ncbiDwnld.py @@ -12,6 +12,7 @@ import time import shutil import urllib +import logging import zipfile import argparse import subprocess @@ -27,13 +28,15 @@ format_path, prep_output, mkOutput, - eprint, - vprint, findExecs, read_json, split_input, + setup_logging, ) from mycotools.lib.dbtools import log_editor, loginCheck, mtdb, read_tax +from pathlib import Path + +logger = logging.getLogger(__name__) pd.options.mode.chained_assignment = None @@ -63,20 +66,20 @@ def prepare_folders(output_path, gff, prot, assem, transcript): file_types = [] if assem: - if not os.path.exists(output_path + "fna"): - os.mkdir(output_path + "fna") + if not Path(output_path + "fna").exists(): + Path(output_path + "fna").mkdir() file_types.append("fna") if gff: - if not os.path.exists(output_path + "gff3"): - os.mkdir(output_path + "gff3") + if not Path(output_path + "gff3").exists(): + Path(output_path + "gff3").mkdir() file_types.append("gff3") if prot: - if not os.path.exists(output_path + "faa"): - os.mkdir(output_path + "faa") + if not Path(output_path + "faa").exists(): + Path(output_path + "faa").mkdir() file_types.append("faa") if transcript: - if not os.path.exists(output_path + "transcript"): - os.mkdir(output_path + "transcript") + if not Path(output_path + "transcript").exists(): + Path(output_path + "transcript").mkdir() file_types.append("transcript") return file_types @@ -85,7 +88,7 @@ def prepare_folders(output_path, gff, prot, assem, transcript): def compile_log(output_path): acc2log = {} - if not os.path.isfile(output_path): + if not Path(output_path).is_file(): with open(output_path, "w") as out: out.write("#acc\tassembly_acc\n") else: @@ -120,7 +123,7 @@ def esearch_ncbi(accession, column, database="assembly"): time.sleep(1) esc_count += 1 else: - print("\tERROR:", accession, "failed to search NCBI") + logger.error(f"{accession} failed to search NCBI") return None return genome_ids @@ -214,7 +217,7 @@ def collect_assembly_accs( if not genome_id: # No IDs retrieved if "ome" in row.keys(): accession = row["ome"] - eprint(spacer + "\t" + accession + " failed to find genome ID", flush=True) + logger.error(spacer + "\t" + accession + " failed to find genome ID") try: failed.append([accession, datetime.strftime(row["version"], "%Y%m%d")]) except TypeError: # if the row can't be formatted as a date @@ -276,7 +279,7 @@ def run_datasets(include, accs_file, output_path, annotated, api=None, verbose=F if annotated: dataset_scaf.append("--annotated") - cwd = os.getcwd() + cwd = str(Path.cwd()) os.chdir(output_path) if verbose: v = None @@ -352,7 +355,7 @@ def parse_datasets(datasets_path, unzip_base, req_files, spacer="\t"): zip_ref.extractall(unzip_base) except zipfile.BadZipFile: return False, False, False - os.remove(datasets_path) + Path(datasets_path).unlink() unzip_path = unzip_base + "ncbi_dataset/" type2ncbi = { @@ -376,8 +379,8 @@ def parse_datasets(datasets_path, unzip_base, req_files, spacer="\t"): if req_files.difference(set(files.keys())): failed.append(data["accession"]) for t, f_ in files.items(): - if os.path.isfile(f_): - os.remove(f_) + if Path(f_).is_file(): + Path(f_).unlink() else: acc2data[data["accession"]] = files @@ -394,7 +397,7 @@ def main( transcript=False, ncbi_df=False, remove=False, - output_path=os.getcwd(), + output_path=str(Path.cwd()), verbose=False, column="assembly_acc", ncbi_column="Assembly", @@ -408,7 +411,7 @@ def main( # assembly, transcript) # check if ncbi_df is a dataframe, and import if not - if not isinstance(ncbi_df, pd.DataFrame) and os.path.isfile(ncbi_df): + if not isinstance(ncbi_df, pd.DataFrame) and Path(ncbi_df).is_file(): ncbi_df = ncbidb2df(ncbi_df) if len(ncbi_df.index) == 0: ncbi_df = pd.DataFrame({i: [v] for i, v in enumerate(list(ncbi_df.keys()))}) @@ -425,7 +428,7 @@ def main( ## CHANGE TO ACCOMODATE BIOSAMPLE/OTHER NCBICOLUMNS if ncbi_column.lower() != "assembly": - vprint(f"{spacer}Assembling NCBI ftp directories", v=verbose, flush=True) + logger.debug(f"{spacer}Assembling NCBI ftp directories") acc2log = compile_log(output_path + "ncbiDwnld.log") acc2log, failed, ncbi_df = collect_assembly_accs( ncbi_df, @@ -474,11 +477,11 @@ def main( count = 0 while count < 3: if not count: - vprint(f"{spacer}Downloading data", v=verbose, flush=True) + logger.debug(f"{spacer}Downloading data") count += 1 else: count += 1 - vprint(f"{spacer}\tAttempt {count}", v=verbose, flush=True) + logger.debug(f"{spacer}\tAttempt {count}") run_datasets( include, @@ -499,9 +502,9 @@ def main( break if acc2files == False and acc2org == False and failed == False: - eprint(f"{spacer}ERROR: ncbiDwnld failed {count} attempts", flush=True) + logger.error(f"{spacer}ncbiDwnld failed {count} attempts") # maybe add a fallback to the old methodology here - eprint(f"{spacer}Consider --fallback", flush=True) + logger.error(f"{spacer}Consider --fallback") sys.exit(10) failed.extend( @@ -510,11 +513,7 @@ def main( # Attempt RefSeq accessions if failed: - vprint( - f"{spacer}Attempting alternative repository for failed downloads", - v=verbose, - flush=True, - ) + logger.debug(f"{spacer}Attempting alternative repository for failed downloads") reattempt_acc = [] for acc in failed: if acc.upper().startswith("GCA"): @@ -563,7 +562,7 @@ def main( ncbi_df.at[acc, tax] = name new_df = pd.concat([new_df, ncbi_df.loc[acc].to_frame().T]) except AttributeError: # multiple entries - eprint(f"{spacer}WARNING: {acc} is redundant", flush=True) + logger.warning(f"{spacer}{acc} is redundant") for acc1, row1 in ncbi_df.loc[acc].iterrows(): for file_t, file_p in acc2files[acc].items(): row1[file_t] = file_p @@ -579,12 +578,7 @@ def main( ncbi_df.at[acc, tax] = name new_df = pd.concat([new_df, ncbi_df.loc[acc].to_frame().T]) except AttributeError: # multiple entries - vprint( - f"{spacer}\tWARNING: {check_acc} is redundant", - v=verbose, - e=True, - flush=True, - ) + logger.debug(f"{spacer}\t{check_acc} is redundant") for acc1, row1 in ncbi_df.loc[acc].iterrows(): for file_t, file_p in acc2files[check_acc].items(): row1[file_t] = file_p @@ -609,7 +603,7 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): records = Entrez.read(handle, validate=False) for record in records: srr = re.search(r'Run acc="(S\w+\d+)"', record["Runs"])[1] - print("\t\t" + srr, flush=True) + logger.info("\t\t" + srr) cmd, count = 1, 0 if pe: while cmd and count < 3: @@ -625,7 +619,7 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): cmd = subprocess.call( [fastqdump, "--split-3", "--gzip", srr], stdout=subprocess.PIPE ) - if os.path.isfile(f"{srr}_1.fastq.gz"): + if Path(f"{srr}_1.fastq.gz").is_file(): # if os.path.isfile(srr + '_1.fastq'): # cmd = subprocess.call(['gzip', f'{srr}_1.fastq']) # cmd = subprocess.call(['gzip', f'{srr}_2.fastq']) @@ -637,9 +631,7 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): ) else: # cmd = subprocess.call(['gzip', f'{srr}.fastq']) - print( - "\t\t\tWARNING: file failed or not paired-end", flush=True - ) + logger.warning("file failed or not paired-end") else: while cmd and count < 3: count += 1 @@ -655,24 +647,24 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): if cmd: continue # cmd = subprocess.call(['gzip', f'{srr}.fastq']) - if os.path.isfile(f"{srr}.fastq.gz"): + if Path(f"{srr}.fastq.gz").is_file(): shutil.move(f"{srr}.fastq.gz", f"{assembly_acc}_{srr}.fq.gz") else: - print("\t\t\tERROR: file failed", flush=True) + logger.error("file failed") -def goSRA(df, output=os.getcwd() + "/", pe=True, column="sra"): +def goSRA(df, output=str(Path.cwd()) + "/", pe=True, column="sra"): print() sra_dir = output + "sra/" - if not os.path.isdir(sra_dir): - os.mkdir(sra_dir) + if not Path(sra_dir).is_dir(): + Path(sra_dir).mkdir() os.chdir(sra_dir) fastqdump = findExecs("fastq-dump", exit={"fastq-dump"}) count = 0 for i, row in df.iterrows(): - print("\t" + row[column], flush=True) + logger.info("\t" + row[column]) get_SRA(row[column], fastqdump[0]) count += 1 if count >= 10: @@ -719,6 +711,7 @@ def cli(): "--fallback", action="store_true", help="Fallback mode if datasets fails" ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.email: ncbi_email = args.email @@ -753,7 +746,7 @@ def cli(): start_time = intro("Download NCBI files", args_dict) if args.sra: - if os.path.isfile(format_path(args.input)): + if Path(format_path(args.input)).is_file(): if not args.column: goSRA( pd.read_csv(format_path(args.input), sep="\t", names=["sra"]), @@ -776,7 +769,7 @@ def cli(): column="sra", ) else: - if os.path.isfile(format_path(args.input)): + if Path(format_path(args.input)).is_file(): ncbi_df = pd.read_csv(args.input, sep="\t", header=None) if not args.column: if "assembly_acc" in ncbi_df.keys(): @@ -853,7 +846,7 @@ def cli(): new_df.to_csv(output + "ncbiDwnld.predb", sep="\t", index=None) if failed: - eprint("ERROR: " + ",".join([str(x[0]) for x in failed]), flush=True) + logger.error(",".join([str(x[0]) for x in failed])) outro(start_time) diff --git a/mycotools/ncbi_dwnld_fallback.py b/mycotools/ncbi_dwnld_fallback.py index c774575..1ed2fe0 100644 --- a/mycotools/ncbi_dwnld_fallback.py +++ b/mycotools/ncbi_dwnld_fallback.py @@ -4,6 +4,7 @@ # NEED to convert to datasets # NEED to consider refseq genomes with annotations when genbank doesn't have them +import logging import os import re import sys @@ -26,11 +27,13 @@ outro, format_path, prep_output, - eprint, - vprint, findExecs, + setup_logging, ) from mycotools.lib.dbtools import log_editor, loginCheck, mtdb, read_tax +from pathlib import Path + +logger = logging.getLogger(__name__) def ncbidb2df(data, stdin=False): @@ -58,20 +61,20 @@ def prepare_folders(output_path, gff, prot, assem, transcript): file_types = [] if assem: - if not os.path.exists(output_path + "fna"): - os.mkdir(output_path + "fna") + if not Path(output_path + "fna").exists(): + Path(output_path + "fna").mkdir() file_types.append("fna") if gff: - if not os.path.exists(output_path + "gff3"): - os.mkdir(output_path + "gff3") + if not Path(output_path + "gff3").exists(): + Path(output_path + "gff3").mkdir() file_types.append("gff3") if prot: - if not os.path.exists(output_path + "faa"): - os.mkdir(output_path + "faa") + if not Path(output_path + "faa").exists(): + Path(output_path + "faa").mkdir() file_types.append("faa") if transcript: - if not os.path.exists(output_path + "transcript"): - os.mkdir(output_path + "transcript") + if not Path(output_path + "transcript").exists(): + Path(output_path + "transcript").mkdir() file_types.append("transcript") return file_types @@ -80,7 +83,7 @@ def prepare_folders(output_path, gff, prot, assem, transcript): def compile_log(output_path, remove=False): acc2log = {} - if not os.path.isfile(output_path + "ncbiDwnld.fallback.log"): + if not Path(output_path + "ncbiDwnld.fallback.log").is_file(): with open(output_path + "ncbiDwnld.fallback.log", "w") as out: out.write( "#ome\tassembly_acc\tassembly\tproteome\tgff3\ttranscript\t" @@ -137,7 +140,7 @@ def esearch_ncbi(accession, column, database="assembly"): time.sleep(1) esc_count += 1 else: - print("\tERROR:", accession, "failed to search NCBI") + logger.error("%s %s %s", "\tERROR:", accession, "failed to search NCBI") return None return genome_ids @@ -234,7 +237,7 @@ def collect_ftps( if not genome_id: # No IDs retrieved if "ome" in row.keys(): accession = row["ome"] - eprint(spacer + "\t" + accession + " failed to find genome ID", flush=True) + logger.info(spacer + "" + accession + " failed to find genome ID") try: failed.append([accession, datetime.strftime(row["version"], "%Y%m%d")]) except TypeError: # if the row can't be formatted as a date @@ -305,10 +308,7 @@ def collect_ftps( ftp_path = str(record_info["FtpPath_GenBank"]) if not ftp_path: - eprint( - spacer + "\t" + new_acc + " failed to return any FTP path", - flush=True, - ) + logger.info(spacer + "" + new_acc + " failed to return any FTP path") try: failed.append( [accession, datetime.strftime(row["version"], "%Y%m%d")] @@ -319,7 +319,7 @@ def collect_ftps( esc_count = 0 ass_md5, gff_md5, trans_md5, prot_md5, md5s = "", "", "", "", {} - basename = os.path.basename(ftp_path) + basename = Path(ftp_path).name dwnld = 0 for attempt in range(3): @@ -357,7 +357,7 @@ def collect_ftps( # data = line.rstrip().split() if data and len(data) == 2: try: - md5s[ftp_path + "/" + os.path.basename(data[1])] = data[ + md5s[ftp_path + "/" + Path(data[1]).name] = data[ 0 ] except IndexError: # 404 error or something else @@ -366,7 +366,7 @@ def collect_ftps( else: md5s = {} - tranname = os.path.basename(ftp_path.replace("/GCA", "/GCF")) + tranname = Path(ftp_path.replace("/GCA", "/GCF")).name # tranname = os.path.basename(ftp_path) assembly = ftp_path + "/" + basename + "_genomic.fna.gz" if assembly in md5s: @@ -469,17 +469,17 @@ def download_files( ftp_link = acc_prots[file_type] dwnlds[file_type] = -1 if file_type == "fna": - file_path = output_dir + "fna/" + os.path.basename(acc_prots[file_type]) + file_path = output_dir + "fna/" + Path(acc_prots[file_type]).name elif file_type == "gff3": - file_path = output_dir + "gff3/" + os.path.basename(acc_prots[file_type]) + file_path = output_dir + "gff3/" + Path(acc_prots[file_type]).name elif file_type == "faa": - file_path = output_dir + "faa/" + os.path.basename(acc_prots[file_type]) + file_path = output_dir + "faa/" + Path(acc_prots[file_type]).name elif file_type == "transcript": file_path = ( - output_dir + "transcript/" + os.path.basename(acc_prots[file_type]) + output_dir + "transcript/" + Path(acc_prots[file_type]).name ) - if os.path.isfile(file_path): + if Path(file_path).is_file(): count += 1 md5_cmd = subprocess.run( ["md5sum", file_path], stdout=subprocess.PIPE @@ -489,13 +489,11 @@ def download_files( md5 = md5_find[0] if md5 == acc_prots[file_type + "_md5"]: - eprint( - f"{spacer}\t{file_type}: {os.path.basename(file_path)}", flush=True - ) + logger.info(f"{spacer}\t{file_type}: {Path(file_path).name}") dwnlds[file_type] = 0 continue - elif os.path.isfile(file_path[:-3]): - eprint(f"{spacer}\t{file_type}: {os.path.basename(file_path)}", flush=True) + elif Path(file_path[:-3]).is_file(): + logger.info(f"{spacer}\t{file_type}: {Path(file_path).name}") dwnlds[file_type] = 0 continue @@ -511,14 +509,14 @@ def download_files( stderr=subprocess.PIPE, ) if not dwnld: - os.rename(file_path + ".tmp", file_path) + Path(file_path + ".tmp").rename(file_path) break else: time.sleep(1) count = 0 if dwnld: - eprint(f"{spacer}\t\tERROR: {file_type} failed", flush=True) + logger.error(f"{spacer}\t\tERROR: {file_type} failed") dwnlds[file_type] = 69 acc_prots[file_type] = "" log_editor( @@ -551,19 +549,19 @@ def download_files( break continue - if not os.path.isfile(file_path): + if not Path(file_path).is_file(): dwnlds[file_type] = 1 - eprint(f"{spacer}\t\tERROR: {file_type} missing", flush=True) + logger.error(f"{spacer}\t\tERROR: {file_type} missing") if remove and file_type in {"fna", "gff3"}: break else: dwnlds[file_type] = 0 - if os.stat(file_path).st_size < 150: - eprint(f"{spacer}\t{file_type}: ERROR, file too small", flush=True) + if Path(file_path).stat().st_size < 150: + logger.error(f"{spacer}\t{file_type}: ERROR, file too small") dwnlds[file_type] = 420 if remove and file_type in {"fna", "gff3"}: break - eprint(f"{spacer}\t{file_type}: {os.path.basename(file_path)}", flush=True) + logger.info(f"{spacer}\t{file_type}: {Path(file_path).name}") return dwnlds, count @@ -599,8 +597,8 @@ def dwnld_mngr_no_MD5( ): run, fail = False, [] for file_type in file_types: - file_path = output_path + file_type + "/" + os.path.basename(data[file_type]) - if not os.path.isfile(file_path): + file_path = output_path + file_type + "/" + Path(data[file_type]).name + if not Path(file_path).is_file(): run = True break @@ -645,7 +643,7 @@ def main( transcript=False, ncbi_df=False, remove=False, - output_path=os.getcwd(), + output_path=str(Path.cwd()), verbose=False, column="assembly_acc", ncbi_column="Assembly", @@ -659,7 +657,7 @@ def main( acc2log = compile_log(output_path, remove) # check if ncbi_df is a dataframe, and import if not - if not isinstance(ncbi_df, pd.DataFrame) and os.path.isfile(ncbi_df): + if not isinstance(ncbi_df, pd.DataFrame) and Path(ncbi_df).is_file(): ncbi_df = ncbidb2df(ncbi_df) if len(ncbi_df.index) == 0: ncbi_df = pd.DataFrame({i: [v] for i, v in enumerate(list(ncbi_df.keys()))}) @@ -673,7 +671,7 @@ def main( ncbi_df = ncbi_df.set_index(pd.Index(list(ncbi_df[column]))) # preserve the original column, but index ncbi_df on it as well - vprint("\n" + spacer + "Assembling NCBI ftp directories", v=verbose, flush=True) + logger.debug("" + spacer + "Assembling NCBI ftp directories") acc2log, failed, ncbi_df = collect_ftps( ncbi_df, acc2log, @@ -694,13 +692,13 @@ def main( } new_df = pd.DataFrame() - vprint(f"\n{spacer}Downloading {len(acc2log)} NCBI files", v=verbose, flush=True) + logger.debug(f"{spacer}Downloading {len(acc2log)} NCBI files") count = 0 if "strain" not in ncbi_df.columns: ncbi_df["strain"] = "" if check_MD5: for acc, data in acc2log.items(): - eprint(spacer + "\t" + str(acc), flush=True) + logger.info(spacer + "" + str(acc)) if data: fail, count = dwnld_mngr( ncbi_df, @@ -717,13 +715,13 @@ def main( failed.append(fail) else: ncbi_df.at[acc, "assemblyPath"] = ( - output_path + "fna/" + os.path.basename(acc2log[acc]["fna"]) + output_path + "fna/" + Path(acc2log[acc]["fna"]).name ) ncbi_df.at[acc, "faa"] = ( - output_path + "faa/" + os.path.basename(acc2log[acc]["faa"]) + output_path + "faa/" + Path(acc2log[acc]["faa"]).name ) ncbi_df.at[acc, "gffPath"] = ( - output_path + "gff3/" + os.path.basename(acc2log[acc]["gff3"]) + output_path + "gff3/" + Path(acc2log[acc]["gff3"]).name ) ncbi_df.at[acc, "genus"] = acc2log[acc]["genus"] ncbi_df.at[acc, "species"] = acc2log[acc]["species"] @@ -741,7 +739,7 @@ def main( else: # there is no checking md5, this is for efficient, so make it # efficient by avoiding conditional expressions for acc, data in acc2log.items(): - eprint(spacer + "\t" + str(acc), flush=True) + logger.info(spacer + "" + str(acc)) fail, count = dwnld_mngr_no_MD5( ncbi_df, data, acc, file_types, output_path, count, remove, api, spacer ) @@ -753,7 +751,7 @@ def main( output_path + file_type + "/" - + os.path.basename(data[file_type]) + + Path(data[file_type]).name ) new_df = pd.concat([new_df, ncbi_df.loc[acc].to_frame().T]) @@ -774,7 +772,7 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): records = Entrez.read(handle, validate=False) for record in records: srr = re.search(r'Run acc="(S\w+\d+)"', record["Runs"])[1] - print("\t\t" + srr, flush=True) + logger.debug("" + srr) cmd, count = 1, 0 if pe: while cmd and count < 3: @@ -790,7 +788,7 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): cmd = subprocess.call( [fastqdump, "--split-3", "--gzip", srr], stdout=subprocess.PIPE ) - if os.path.isfile(srr + "_1.fastq"): + if Path(srr + "_1.fastq").is_file(): # if os.path.isfile(srr + '_1.fastq'): # cmd = subprocess.call(['gzip', f'{srr}_1.fastq']) # cmd = subprocess.call(['gzip', f'{srr}_2.fastq']) @@ -802,9 +800,7 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): ) else: # cmd = subprocess.call(['gzip', f'{srr}.fastq']) - print( - "\t\t\tWARNING: file failed or not paired-end", flush=True - ) + logger.warning("file failed or not paired-end") else: while cmd and count < 3: count += 1 @@ -820,20 +816,20 @@ def get_SRA(assembly_acc, fastqdump="fastq-dump", pe=True): if cmd: continue # cmd = subprocess.call(['gzip', f'{srr}.fastq']) - if os.path.isfile(srr + ".fastq.gz"): + if Path(srr + ".fastq.gz").is_file(): shutil.move( srr + ".fastq.gz", assembly_acc + "_" + srr + ".fq.gz" ) else: - print("\t\t\tERROR: file failed", flush=True) + logger.error("file failed") -def goSRA(df, output=os.getcwd() + "/", pe=True): +def goSRA(df, output=str(Path.cwd()) + "/", pe=True): print() sra_dir = output + "sra/" - if not os.path.isdir(sra_dir): - os.mkdir(sra_dir) + if not Path(sra_dir).is_dir(): + Path(sra_dir).mkdir() os.chdir(sra_dir) fastqdump = findExecs("fastq-dump", exit=set("fastq-dump")) count = 0 @@ -844,7 +840,7 @@ def goSRA(df, output=os.getcwd() + "/", pe=True): row_key = "assembly_acc" for i, row in df.iterrows(): - print("\t" + row[row_key], flush=True) + logger.debug("" + row[row_key]) get_SRA(row[row_key], fastqdump[0]) count += 1 if count >= 10: @@ -888,6 +884,7 @@ def cli(): parser.add_argument("-e", "--email", help="NCBI email") parser.add_argument("--api", help="NCBI API key for high query rate") args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.email: ncbi_email = args.email @@ -904,7 +901,7 @@ def cli(): Entrez.api_key = ncbi_api if not args.output: - output = os.getcwd() + "/" + output = str(Path.cwd()) + "/" else: output = format_path(args.output) @@ -924,14 +921,14 @@ def cli(): # sys.exit( 37 ) if args.sra: - if os.path.isfile(format_path(args.input)): + if Path(format_path(args.input)).is_file(): goSRA( pd.read_csv(format_path(args.input), sep="\t"), output, pe=args.paired ) else: goSRA(pd.DataFrame({"sra": [args.input.rstrip()]}), output, pe=args.paired) else: - if os.path.isfile(format_path(args.input)): + if Path(format_path(args.input)).is_file(): ncbi_df = pd.read_csv(args.input, sep="\t", header=None) if not args.column: if "assembly_acc" in ncbi_df.keys(): diff --git a/mycotools/ome2name.py b/mycotools/ome2name.py index d50b9ea..ff49323 100755 --- a/mycotools/ome2name.py +++ b/mycotools/ome2name.py @@ -1,10 +1,13 @@ #! /usr/bin/env python3 -import os import re import sys -from mycotools.lib.kontools import format_path, sys_start, eprint +import logging +from mycotools.lib.kontools import format_path, sys_start, setup_logging from mycotools.lib.dbtools import primaryDB, mtdb +from pathlib import Path + +logger = logging.getLogger(__name__) def parse_args(args): @@ -32,7 +35,7 @@ def parse_args(args): if tax: rank = arg.lower() if rank not in valid_ranks: - eprint("\nERROR: --taxonomy not in " + str(valid_ranks), flush=True) + logger.error("--taxonomy not in " + str(valid_ranks)) sys.exit(10) tax = False elif "-" in arg: @@ -40,11 +43,11 @@ def parse_args(args): tax = True continue else: - eprint("\nERROR: invalid argument `-`", flush=True) + logger.error("invalid argument `-`") sys.exit(11) - elif os.path.isfile(format_path(arg)): + elif Path(format_path(arg)).is_file(): if not go_on: - eprint("\nERROR: multiple files", flush=True) + logger.error("multiple files") db = mtdb(arg) go_on = False continue @@ -157,6 +160,7 @@ def main( def cli(): """Command line entry point""" + setup_logging() usage = ( "USAGE: ome2name | ome2name.py " + " [.mtdb] asvg*&\nDEFAULTS: master db, see script for default" diff --git a/mycotools/predb2mtdb.py b/mycotools/predb2mtdb.py index 1eeeb1e..482ec75 100755 --- a/mycotools/predb2mtdb.py +++ b/mycotools/predb2mtdb.py @@ -3,7 +3,7 @@ # NEED source to reference the annotation source # NEED to error check FAA generation simply by file size -import os +import logging import re import sys import copy @@ -11,7 +11,7 @@ import multiprocessing as mp from tqdm import tqdm from collections import Counter, defaultdict -from mycotools.lib.kontools import gunzip, mkOutput, format_path, eprint, vprint +from mycotools.lib.kontools import gunzip, mkOutput, format_path from mycotools.lib.biotools import ( gff2list, list2gff, @@ -27,6 +27,9 @@ from mycotools.utils.gff2gff3 import main as gff2gff3 from mycotools.utils.curGFF3 import rename_and_organize as rename_and_organize from mycotools.gff2seq import aamain as gff2seq +from pathlib import Path + +logger = logging.getLogger(__name__) predb_headers = [ "assembly_accession", @@ -47,7 +50,7 @@ def acq_forbid_omes(file_path): """Parse a file with forbidden ome accessions - ome codes that have been used before and are no longer valid""" - if not os.path.isfile(file_path): + if not Path(file_path).is_file(): return set() with open(file_path, "r") as raw: relics = set([x.rstrip() for x in raw]) @@ -59,8 +62,8 @@ def prep_output(base_dir): wrk_dir = out_dir + "working/" dirs = [out_dir, wrk_dir, wrk_dir + "gff3/", wrk_dir + "fna/", wrk_dir + "faa/"] for dir_ in dirs: - if not os.path.isdir(dir_): - os.mkdir(dir_) + if not Path(dir_).is_dir(): + Path(dir_).mkdir() return dirs[:2] @@ -74,7 +77,7 @@ def copy_file(old_path, new_path): def move_biofile(old_path, ome, typ, wrk_dir, suffix=""): if old_path.endswith(".gz"): - if not os.path.isfile(old_path[:-3]): + if not Path(old_path[:-3]).is_file(): temp_path = gunzip(old_path) new_path = wrk_dir + ome + "." + typ + suffix else: @@ -83,9 +86,9 @@ def move_biofile(old_path, ome, typ, wrk_dir, suffix=""): copy_file(format_path(temp_path), new_path) else: new_path = wrk_dir + ome + "." + typ + suffix - if not os.path.isfile(new_path) and os.path.isfile(old_path): + if not Path(new_path).is_file() and Path(old_path).is_file(): copy_file(format_path(old_path), new_path) - elif not os.path.isfile(new_path): + elif not Path(new_path).is_file(): raise IOError(old_path, new_path) else: copy_file(format_path(old_path), new_path) @@ -108,15 +111,12 @@ def gen_predb(): "no", "2018", ] - eprint( - 'INSTRUCTIONS: fill in each column with the relevant information and \ + logger.info('INSTRUCTIONS: fill in each column with the relevant information and \ separate each column by a tab. The predb can be filled in \ via spreadsheet software and exported as a tab delimited `.tsv`. \ ASSEMBLY ACCESSIONS and PREVIOUS_OME fields must be unique to the \ genome; otherwise predb2mtdb will update the corresponding database entry. \ - Novel data must be filled in as "new" for the genomeSource column.', - flush=True, - ) + Novel data must be filled in as "new" for the genomeSource column.') outputStr = "#" + "\t".join(predb_headers) outputStr += "\n#" + "\t".join(example) + "\n" @@ -183,11 +183,8 @@ def read_predb(predb_path, spacer="\t"): # required_headers.remove(head) missing_headers = required_headers.difference(set(i2header.values())) if missing_headers: - eprint( - f"{spacer}ERROR: Required columns missing: " - + f"{missing_headers}", - flush=True, - ) + logger.error(f"{spacer}ERROR: Required columns missing: " + + f"{missing_headers}") sys.exit(4) # if not headers: # predb = {x: [] for x in line.rstrip()[1:].split('\t')} @@ -197,11 +194,8 @@ def read_predb(predb_path, spacer="\t"): # proceed with default header organization scheme if not i2header: if len(entry) != len(predb_headers): - eprint( - spacer + "ERROR: Incorrect columns, line " + str(i), - flush=True, - ) - eprint(predb_headers, "\n", entry, flush=True) + logger.error(spacer + "Incorrect columns, line " + str(i)) + logger.debug("%s\n%s", predb_headers, entry) sys.exit(3) for i1, v in enumerate(entry): predb[predb_headers[i1]].append(v.rstrip()) @@ -254,10 +248,7 @@ def read_predb(predb_path, spacer="\t"): x.lower() not in {"y", "n", "yes", "no", "", "true", "false"} for x in predb["restriction"] ): - eprint( - spacer + "ERROR: useRestriction entries must be in {y, n, yes, no}", - flush=True, - ) + logger.error(spacer + "useRestriction entries must be in {y, n, yes, no}") sys.exit(4) except KeyError: if not "restriction" in predb and "published" not in predb: @@ -266,17 +257,12 @@ def read_predb(predb_path, spacer="\t"): predb["restriction"] = [bool(x) for x in predb["published"]] if any(x.lower() not in {"jgi", "ncbi", "new"} for x in predb["source"]): - eprint( - [ + logger.info([ predb["assembly_acc"][i] for i, v in enumerate(predb["source"]) if v not in {"jgi", "ncbi", "new"} - ] - ) - eprint( - spacer + "ERROR: genomeSource entries must be in {jgi, ncbi, new}", - flush=True, - ) + ]) + logger.error(spacer + "genomeSource entries must be in {jgi, ncbi, new}") sys.exit(5) missing_from_predb = list(set(predb_headers).difference(set(predb.keys()))) @@ -384,7 +370,7 @@ def gen_omes(newdb, refdb=None, ome_col="ome", forbidden=set(), spacer="\t"): new_ome = re.sub(r"\.\d+$", "." + str(v), ome) else: new_ome = ome + ".1" # first modified version - eprint(spacer + ome + " update -> " + new_ome, flush=True) + logger.info(spacer + ome + " update -> " + new_ome) newdb["ome"][i] = new_ome continue @@ -393,19 +379,13 @@ def gen_omes(newdb, refdb=None, ome_col="ome", forbidden=set(), spacer="\t"): except TypeError: todel.append(i) if not isinstance(newdb["assembly_acc"][i], float): - eprint( - spacer + logger.info(spacer + newdb["assembly_acc"][i] + " no metadata - " - + "failed", - flush=True, - ) + + "failed") elif "index" in newdb: # for updateDB if not isinstance(newdb, float): - eprint( - spacer + newdb["index"][i] + " no metadata - " + "failed", - flush=True, - ) + logger.info(spacer + newdb["index"][i] + " no metadata - " + "failed") continue else: # no use appending failed when there's no identifiable # info @@ -442,7 +422,7 @@ def gen_omes(newdb, refdb=None, ome_col="ome", forbidden=set(), spacer="\t"): new_ome = re.sub(r"\.\d+$", "." + str(v), ome) else: new_ome = ome + ".1" # first modified version - eprint(spacer + ome + " update -> " + new_ome, flush=True) + logger.info(spacer + ome + " update -> " + new_ome) newdb["ome"][i] = new_ome elif ome in refdb_nover: # has a version, wasn't given in predb version_ome = refdb_nover[ome] @@ -452,7 +432,7 @@ def gen_omes(newdb, refdb=None, ome_col="ome", forbidden=set(), spacer="\t"): new_ome = ome + "." + str(v) else: raise TypeError("unknown error " + ome) - eprint(spacer + ome + " version added -> " + new_ome, flush=True) + logger.info(spacer + ome + " version added -> " + new_ome) newdb["ome"][i] = new_ome for i in reversed(todel): @@ -506,23 +486,20 @@ def cur_mngr( verbose=False, ): - predb_dir = os.path.basename(os.path.dirname(wrk_dir[:-1])) + "/working/" + predb_dir = Path(str(Path(wrk_dir[:-1]).parent)).name + "/working/" # assembly FNAs - vprint("\t" + ome, v=verbose, flush=True) + logger.debug("" + ome) uncur_fna_path = wrk_dir + "fna/" + ome + ".fna.uncur" cur_fna_path = wrk_dir + "fna/" + ome + ".fna" - vprint("\t\t" + predb_dir + "fna/" + ome + ".fna", v=verbose, flush=True) - if not os.path.isfile(cur_fna_path): + logger.debug("" + predb_dir + "fna/" + ome + ".fna") + if not Path(cur_fna_path).is_file(): try: uncur_fna_path = move_biofile( raw_fna_path, ome, "fa", wrk_dir + "fna/", suffix=".uncur" ) except IOError as ie: - eprint( - spacer + ome + "|" + assembly_accession + " failed FNA parsing", - flush=True, - ) + logger.info(spacer + ome + "|" + assembly_accession + " failed FNA parsing") if exit: raise ie from None return ome, False, "fna" @@ -531,17 +508,14 @@ def cur_mngr( # gene coordinate GFF3s uncur_gff_path = wrk_dir + "gff3/" + ome + ".gff3.uncur" cur_gff_path = wrk_dir + "gff3/" + ome + ".gff3" - vprint("\t\t" + predb_dir + "gff3/" + ome + ".gff3", v=verbose, flush=True) - if not os.path.isfile(cur_gff_path): + logger.debug("" + predb_dir + "gff3/" + ome + ".gff3") + if not Path(cur_gff_path).is_file(): try: uncur_gff_path = move_biofile( raw_gff_path, ome, "gff3", wrk_dir + "gff3/", suffix=".uncur" ) except IOError as ie: - eprint( - spacer + ome + "|" + assembly_accession + " failed GFF3 parsing", - flush=True, - ) + logger.info(spacer + ome + "|" + assembly_accession + " failed GFF3 parsing") if exit: raise ie from None return ome, False, "gff3" @@ -553,18 +527,15 @@ def cur_mngr( try: gff_mngr(ome, gff, cur_gff_path, source, assembly_accession) except Exception as e: # catch all errors to continue script - eprint( - spacer + ome + "|" + assembly_accession + " failed GFF3 curation", - flush=True, - ) + logger.info(spacer + ome + "|" + assembly_accession + " failed GFF3 curation") if exit: raise e from None return ome, False, "gff3" # proteome FAAs faa_path = wrk_dir + "faa/" + ome + ".faa" - vprint("\t\t" + predb_dir + "faa/" + ome + ".faa", v=verbose, flush=True) - if not os.path.isfile(faa_path): + logger.debug("" + predb_dir + "faa/" + ome + ".faa") + if not Path(faa_path).is_file(): try: faa = gff2seq(gff2list(cur_gff_path), fa2dict(cur_fna_path), spacer=spacer) # raise a value error if there is not a sequence for all predicted @@ -573,16 +544,10 @@ def cur_mngr( if faa and len(missing_seq) == len(faa): raise ValueError("no sequences generated in proteome") elif missing_seq: - eprint( - f"{spacer}\tWARNING: {len(missing_seq)} " - + "CDSs translated blank sequences", - flush=True, - ) + logger.warning(f"{spacer}\tWARNING: {len(missing_seq)} " + + "CDSs translated blank sequences") except Exception as e: # catch all errors - eprint( - spacer + ome + "|" + assembly_accession + " failed proteome generation", - flush=True, - ) + logger.info(spacer + ome + "|" + assembly_accession + " failed proteome generation") if exit: raise e return ome, False, "faa" @@ -591,18 +556,18 @@ def cur_mngr( shutil.move(faa_path + ".tmp", faa_path) if remove: - if os.path.isfile(uncur_gff_path): - os.remove(uncur_gff_path) - if os.path.isfile(raw_gff_path): - os.remove(raw_gff_path) - if os.path.isfile(re.sub(r"\.gz$", "", raw_gff_path)): - os.remove(re.sub(r"\.gz$", "", raw_gff_path)) - if os.path.isfile(uncur_fna_path): - os.remove(uncur_fna_path) - if os.path.isfile(raw_fna_path): - os.remove(raw_fna_path) - if os.path.isfile(re.sub(r"\.gz$", "", raw_fna_path)): - os.remove(re.sub(r"\.gz$", "", raw_fna_path)) + if Path(uncur_gff_path).is_file(): + Path(uncur_gff_path).unlink() + if Path(raw_gff_path).is_file(): + Path(raw_gff_path).unlink() + if Path(re.sub(r"\.gz$", "", raw_gff_path)).is_file(): + Path(re.sub(r"\.gz$", "", raw_gff_path)).unlink() + if Path(uncur_fna_path).is_file(): + Path(uncur_fna_path).unlink() + if Path(raw_fna_path).is_file(): + Path(raw_fna_path).unlink() + if Path(re.sub(r"\.gz$", "", raw_fna_path)).is_file(): + Path(re.sub(r"\.gz$", "", raw_fna_path)).unlink() return ome, cur_fna_path, cur_gff_path, faa_path @@ -699,11 +664,11 @@ def main( ): for dir_ in [wrk_dir + "fna/", wrk_dir + "gff3/", wrk_dir + "faa/"]: - if not os.path.isdir(dir_): - os.mkdir(dir_) + if not Path(dir_).is_dir(): + Path(dir_).mkdir() infdb = predb2mtdb(predb) - vprint("\nGenerating omes", v=verbose, flush=True) + logger.debug("Generating omes") omedb, failed = gen_omes( infdb, refdb, ome_col="ome", forbidden=forbidden, spacer=spacer ) @@ -726,7 +691,7 @@ def main( ] ) - vprint("\nCurating data", v=verbose, flush=True) + logger.debug("Curating data") if cpus > 1: with mp.Pool(processes=cpus) as pool: cur_data = pool.starmap(cur_mngr, tqdm(cur_cmds, total=len(cur_cmds))) @@ -757,7 +722,7 @@ def cli(): ) if any(x in {"-h", "--help", "-help"} for x in sys.argv): - eprint("\n" + usage + "\n", flush=True) + logger.info("" + usage + "\n") sys.exit(0) elif len(sys.argv) == 1: print(gen_predb()) @@ -783,9 +748,9 @@ def cli(): # if ncbi_api: # Entrez.api_key = ncbi_api - eprint("\nPreparing run", flush=True) + logger.info("Preparing run") predb = read_predb(format_path(sys.argv[1]), spacer="\t") - out_dir, wrk_dir = prep_output(os.path.dirname(format_path(sys.argv[1]))) + out_dir, wrk_dir = prep_output(str(Path(format_path(sys.argv[1])).parent)) forbid_omes = acq_forbid_omes(file_path=format_path("$MYCODB/../log/relics.txt")) diff --git a/mycotools/treetools.py b/mycotools/treetools.py index a2d26d7..c6976d6 100755 --- a/mycotools/treetools.py +++ b/mycotools/treetools.py @@ -1,20 +1,23 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import argparse from itertools import chain from cogent3 import PhyloNode, load_tree -from mycotools.lib.kontools import format_path, split_input, eprint, vprint +from mycotools.lib.kontools import format_path, split_input, setup_logging from mycotools.lib.dbtools import mtdb +from pathlib import Path + +logger = logging.getLogger(__name__) def compile_tree(tree_path, root=[], verbose=False): """Compile a phylogeny from a path and conver the tip names to an index""" phylo = load_tree(tree_path) tips = set(phylo.get_tip_names()) - vprint(f"{len(tips)} tips on input", v=verbose, e=True) + logger.debug(f"{len(tips)} tips on input") if len(root) == 1: phylo = phylo.rooted_with_tip(root[0]) @@ -27,9 +30,7 @@ def compile_tree(tree_path, root=[], verbose=False): mrca_tip_len = min([v[1] for v in list(nodes.values())]) mrca_edge = [k for k, v in nodes.items() if v[1] == mrca_tip_len] if mrca_edge[0] == "root": - eprint( - f"WARNING: rooting with {root} does not change current tree", flush=True - ) + logger.warning(f"rooting with {root} does not change current tree") phylo = phylo.rooted_at(mrca_edge[0]) return phylo @@ -39,7 +40,7 @@ def prune_to_new(phylo, tips=[], spacer=""): """Prune missing tips from a rooted phylogeny""" missing = set(phylo.get_tip_names()).difference(tips) if missing: - eprint(f"{spacer}Removing {len(missing)} tips", flush=True) + logger.info(f"{spacer}Removing {len(missing)} tips") todel = [] for n in phylo.tips(): if n.name in missing: @@ -97,13 +98,14 @@ def cli(): help="Omit support values", ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) root = split_input(args.root) if args.tips: tips_path = format_path(args.tips) - if os.path.isfile(tips_path): - eprint("\nDetecting path input", flush=True) + if Path(tips_path).is_file(): + logger.info("Detecting path input") with open(tips_path, "r") as raw: tips = list(chain(*[line.rstrip().split() for line in raw])) else: @@ -114,8 +116,8 @@ def cli(): convert = {} if args.convert: convert_path = format_path(args.convert) - if os.path.isfile(convert_path): - eprint("\nDetecting path input", flush=True) + if Path(convert_path).is_file(): + logger.info("Detecting path input") with open(convert_path, "r") as raw: for line in raw: data = line.rstrip() @@ -123,7 +125,7 @@ def cli(): k2v = data.split() convert[k2v[0]] = k2v[1] else: - eprint("\nERROR: --convert must be a valid reference file", flush=True) + logger.error("--convert must be a valid reference file") if args.mtdb: db = mtdb(format_path(args.mtdb)) @@ -136,7 +138,7 @@ def cli(): format_path(args.input), db, tips, trim=args.prune, root=root, convert=convert ) - eprint(f"{len(phylo.get_tip_names())} tips on output", flush=True) + logger.info(f"{len(phylo.get_tip_names())} tips on output") p_str = phylo.get_newick(with_distances=True) if args.support: print(p_str, flush=True) diff --git a/mycotools/update_mtdb.py b/mycotools/update_mtdb.py index f1e72fa..ecb2661 100755 --- a/mycotools/update_mtdb.py +++ b/mycotools/update_mtdb.py @@ -12,6 +12,7 @@ # will require logging whatever NCBI omes directly overlap MycoCosm # NEED to remove overlap when rerunning failed genomes +import logging import os import re import sys @@ -46,13 +47,14 @@ intro, outro, format_path, - eprint, prep_output, collect_files, read_json, write_json, split_input, findExecs, + setup_logging, + atomic_write, ) from mycotools.lib.biotools import fa2dict, gff2list, dict2fa, list2gff from mycotools.ncbiDwnld import ( @@ -69,6 +71,27 @@ from mycotools.predb2mtdb import predb_headers, read_predb, gen_omes from mycotools.assemblyStats import main as assStats from mycotools.annotationStats import main as annStats +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def _read_ledger(file_path, comment="#"): + """Return the meaningful lines of a ledger/sidecar file. + + Yields the shared read half of the ledger parsers below: returns a list of + right-stripped lines, skipping comment lines (those beginning with + `comment`) and lines that are empty once stripped. Returns an empty list if + the file does not exist, so callers can treat a missing ledger as empty. + """ + if not Path(file_path).is_file(): + return [] + with open(file_path, "r") as raw: + return [ + line.rstrip() + for line in raw + if not line.startswith(comment) and line.strip() + ] def validate_t_and_c(config, discrepancy=False): @@ -89,15 +112,12 @@ def validate_t_and_c(config, discrepancy=False): # if there isnt a configuration, alert the user to use-restriction policies else: - print( - "\nPlease review JGI use-restricted data policy here: " + logger.debug("Please review JGI use-restricted data policy here: " + "https://jgi.doe.gov/user-programs/pmo-overview/policies/" + "\nPlease review GenBank use-restricted data policy here: " + "https://ncbi.nlm.nih.gov/genbank/" + "\nPlease review how Mycotools handles use-restricted data here:" - + " https://github.com/xonq/mycotools/blob/master/MTDB.md", - flush=True, - ) + + " https://github.com/xonq/mycotools/blob/master/MTDB.md") check = "" if check.lower() not in {"y", "yes"}: check = input( @@ -112,7 +132,7 @@ def validate_t_and_c(config, discrepancy=False): + "\n\nPlease type [y]es/[N]o if you acknowledge these terms: " ) if check.lower() not in {"y", "yes"}: - print("\nRerun without --nonpublished", flush=True) + logger.info("Rerun without --nonpublished") sys.exit(1) nonpublished = "yes" @@ -176,8 +196,8 @@ def initDB( if not output.endswith("/"): output += "/" for new_dir in new_dirs: - if not os.path.isdir(new_dir): - os.mkdir(new_dir) + if not Path(new_dir).is_dir(): + Path(new_dir).mkdir() config = gen_config( branch=branch, @@ -193,7 +213,7 @@ def initDB( if not rogue: # this is a relic, and needs to be adjusted to a central reference if # that is ever created - if not os.path.isdir(init_dir + "mtdb"): + if not Path(init_dir + "mtdb").is_dir(): # NEED TO CHANGE FROM SSH TO LINK ONCE OPEN (config['repository']) git_exit = subprocess.call( [ @@ -205,44 +225,23 @@ def initDB( ] ) if git_exit != 0: - eprint("\nERROR: git clone failed.", flush=True) + logger.error("git clone failed.") sys.exit(2) else: - print("\nmycotoolsdb directory already exists", flush=True) + logger.info("mycotoolsdb directory already exists") # NEED TO ADD GITIGNORE TO GIT if not primaryDB(): - eprint( - "\nERROR: no YYYYmmdd.mtdb in " + format_path(envs["MYCODB"]), - flush=True, - ) + logger.error("no YYYYmmdd.mtdb in " + format_path(envs["MYCODB"])) sys.exit(3) else: new_db_path = output + "mtdb/" + date + ".mtdb" - if not os.path.isfile(new_db_path): + if not Path(new_db_path).is_file(): with open(output + "mtdb/" + date + ".mtdb", "w") as out: out.write("".join(["\t" for x in mtdb.columns])) return output, config -def parse_forbidden(forbidden_path): - """Read the log file containing information on forbidden genomes""" - # NEED to be rewritten to reference a central repository forbidden file - - log_path = format_path(forbidden_path) - if os.path.isfile(log_path): - log_dict = readLog(log_path) - else: - log_dict = {} - - return log_dict - - -def add_forbidden(tag, source, file_path=None, flag="failed download"): - edit = tag + "\t" + source + "\t" + flag - log_editor(file_path, tag, edit) - - def parse_dups(file_path): """Retrieve a file containing replicated genomes and ignore these. This is important for dereplication of discrepant genus naming between NCBI @@ -252,14 +251,10 @@ def parse_dups(file_path): this is also to some extent present in NCBI. Ultimately, a manually curated file is necessary for this and should be held in a central repository.""" duplicates = {} - if os.path.isfile(file_path): - with open(file_path, "r") as raw: - for line in raw: - if not line.startswith("#"): - data = [x.rstrip() for x in line.split("\t") if x] - if data: - duplicates[data[0]] = [data[1], data[2], data[3]] - + for line in _read_ledger(file_path): + data = [x.rstrip() for x in line.split("\t") if x] + if data: + duplicates[data[0]] = [data[1], data[2], data[3]] return duplicates @@ -273,87 +268,66 @@ def parse_dups(file_path): def acq_forbid_omes(file_path): """Parse a file with forbidden ome accessions - ome codes that have been used before and are no longer valid""" - if not os.path.isfile(file_path): - return set() - with open(file_path, "r") as raw: - relics = set([x.rstrip() for x in raw]) - return relics + return set(_read_ledger(file_path)) def write_forbid_omes(omes, file_path): """Add to a file of forbidden ome accessions so that these are not ever used again, even if the codename is removed from the database""" - if os.path.isfile(file_path): - with open(file_path, "r") as raw: - old_relics = set([x.rstrip() for x in raw]) - new_relics = old_relics.union(set(omes)) - else: - new_relics = set(omes) - - with open(file_path + ".tmp", "w") as out: # be cautious because if it - # cancels then we lose the old data + # union with any pre-existing relics; atomic_write guards against losing + # the old data if the write is cancelled midway + new_relics = set(_read_ledger(file_path)).union(set(omes)) + with atomic_write(file_path) as out: out.write("\n".join([str(x) for x in sorted(new_relics)])) - shutil.move(file_path + ".tmp", file_path) def parse_failed(file_path=None, rerun=False): """Parse a file that stores the failed accessions and metadata of the attempted acquisition. Return a dictionary that contains the failed accession and its metadata.""" - prev_failed = {} - if not os.path.isfile(file_path) or rerun: + if not Path(file_path).is_file() or rerun: with open(file_path, "w") as out: out.write("#code\tsource\tversion\tattempt_date") - else: - with open(file_path, "r") as raw: - for line in raw: - if not line.startswith("#"): - data = [x.rstrip() for x in line.split("\t")] - while len(data) < 4: - data.append("") - prev_failed[data[0]] = { - "source": data[1], - "version": data[2], - "attempt_date": data[3], - } - + return {} + prev_failed = {} + for line in _read_ledger(file_path): + data = [x.rstrip() for x in line.split("\t")] + while len(data) < 4: + data.append("") + prev_failed[data[0]] = { + "source": data[1], + "version": data[2], + "attempt_date": data[3], + } return prev_failed def parse_jgi2ncbi(file_path): """Parse previously collected NCBI to JGI data to limit querying""" - jgi2ncbi = {} - if not os.path.isfile(file_path): + if not Path(file_path).is_file(): with open(file_path, "w") as out: out.write("#ncbi_acc\tmycocosm_portal") - else: - with open(file_path, "r") as raw: - for line in raw: - if not line.startswith("#"): - d = line.rstrip().split("\t") - ncbi, jgi = d[0], d[1].lower() - jgi2ncbi[jgi] = ncbi - + return {} + jgi2ncbi = {} + for line in _read_ledger(file_path): + d = line.split("\t") + jgi2ncbi[d[1].lower()] = d[0] return jgi2ncbi def parse_true_ncbi(file_path): """Parse accessions considered to be unique to NCBI""" - true_ncbi = set() - if not os.path.isfile(file_path): + if not Path(file_path).is_file(): with open(file_path, "w") as out: out.write("#ncbi_acc") - else: - with open(file_path, "r") as raw: - true_ncbi = set([x.rstrip() for x in raw if not x.startswith("#")]) - - return true_ncbi + return set() + return set(_read_ledger(file_path)) def add_true_ncbi(true_ncbi, file_path=None): """Add to a ledger of accessions considered to be unique to NCBI""" - with open(file_path, "w") as out: - out.write("#ncbi_acc\n" + "\n".join([str(x) for x in list(true_ncbi)])) + with atomic_write(file_path) as out: + out.write("#ncbi_acc\n" + "\n".join([str(x) for x in true_ncbi])) def add_jgi2ncbi(jgi2ncbi, file_path=None): @@ -361,7 +335,7 @@ def add_jgi2ncbi(jgi2ncbi, file_path=None): is prone to failure given that the field JGI uses to supply their genome accession is either absent from some NCBI entries, or is in a different field""" - with open(file_path, "w") as out: + with atomic_write(file_path) as out: out.write("#ncbi_acc\tmycocosm_portal\n") for jgi, ncbi in jgi2ncbi.items(): out.write(ncbi + "\t" + jgi + "\n") @@ -388,7 +362,7 @@ def dwnld_mycocosm( check_curl = findExecs(["curl"], verbose=False) - if not os.path.isfile(out_file): + if not Path(out_file).is_file(): for attempt in range(3): if check_curl: curl_cmd = subprocess.call( @@ -399,7 +373,7 @@ def dwnld_mycocosm( shutil.move(out_file + ".tmp", out_file) break if curl_cmd: - eprint("\nERROR: failed to retrieve MycoCosm table", flush=True) + logger.error("failed to retrieve MycoCosm table") else: resp = requests.get(url) with open(out_file + ".tmp", "wb") as f: @@ -426,7 +400,7 @@ def dwnld_ncbi_metadata( eukaryotes, and return a Pandas dataframe""" ncbi_url = ncbi_url + group + ".txt" - if not os.path.isfile(ncbi_file): + if not Path(ncbi_file).is_file(): getTbl = subprocess.call(["curl", ncbi_url, "-o", ncbi_file + ".tmp"]) shutil.move(ncbi_file + ".tmp", ncbi_file) ncbi_df = pd.read_csv(ncbi_file, sep="\t") @@ -442,20 +416,20 @@ def prep_taxa_cols( gca_prep = [x.upper().replace("GCF", "GCA") for x in skip_prep] gcf_prep = [x.upper().replace("GCA", "GCF") for x in skip_prep] skip = set(gca_prep + gcf_prep) - if not os.path.isdir(taxonomy_dir): - os.mkdir(taxonomy_dir) + if not Path(taxonomy_dir).is_dir(): + Path(taxonomy_dir).mkdir() aa_file = taxonomy_dir + "assembly_accs.genbank.txt" with open(aa_file, "w") as out: out.write("\n".join([x for x in list(df["assembly_acc"]) if x not in skip])) attempts, datasets_cmd = 0, 0 - if not os.path.isdir(taxonomy_dir + "ncbi_dataset"): + if not Path(taxonomy_dir + "ncbi_dataset").is_dir(): datasets_path = taxonomy_dir + "ncbi_dataset.zip" while attempts < max_attempts: if attempts: - eprint("\t\t\tReattempting", flush=True) - if os.path.isfile(datasets_path): - os.remove(datasets_path) + logger.info("Reattempting") + if Path(datasets_path).is_file(): + Path(datasets_path).unlink() attempts += 1 datasets_cmd = run_datasets( None, aa_file, taxonomy_dir, True, api=api, verbose=True @@ -463,35 +437,29 @@ def prep_taxa_cols( try: with zipfile.ZipFile(datasets_path, "r") as zip_ref: zip_ref.extractall(taxonomy_dir) - os.remove(datasets_path) + Path(datasets_path).unlink() break except zipfile.BadZipFile: - eprint( - f"\t\tERROR: datasets download corrupted - {attempts}", flush=True - ) + logger.error(f"datasets download corrupted - {attempts}") if attempts == max_attempts: sys.exit(11) except FileNotFoundError: - eprint(f"\t\tERROR: datasets failed - {attempts}", flush=True) + logger.error(f"datasets failed - {attempts}") if datasets_cmd: - eprint(f"\t\tWARNING: datasets failed, assuming no genomes found", flush=True) + logger.warning(f"datasets failed, assuming no genomes found") acc2org_n, acc2meta = {}, {} else: acc2org_n, acc2meta, org_failed = compile_organism_names( taxonomy_dir + "ncbi_dataset/" ) - print( - f"\t\t{len(acc2meta) + len(org_failed)}", - "genomes queried from GenBank", - flush=True, - ) - print(f'\t\t{len(org_failed)/len(df["assembly_acc"])*100}% failed', flush=True) + logger.info("%s %s", f"\t\t{len(acc2meta) + len(org_failed)}", "genomes queried from GenBank") + logger.debug(f'\t\t{len(org_failed)/len(df["assembly_acc"])*100}% failed') # check for RefSeq for failed entries refseq_dir = taxonomy_dir + "refseq/" - if not os.path.isdir(refseq_dir): - os.mkdir(refseq_dir) + if not Path(refseq_dir).is_dir(): + Path(refseq_dir).mkdir() missing_accs = sorted( set(df["assembly_acc"]).difference( set(acc2org_n.keys()).union(set(acc2org.keys())) @@ -510,15 +478,15 @@ def prep_taxa_cols( # attempt to download the ncbi_datasets zip file until allowed attempts are # exhausted rs_datasets_cmd = 0 - if not os.path.isdir(refseq_dir + "ncbi_dataset"): - print(f"\t\tChecking RefSeq for {len(reattempt_acc)} entries", flush=True) + if not Path(refseq_dir + "ncbi_dataset").is_dir(): + logger.debug(f"Checking RefSeq for {len(reattempt_acc)} entries") rs_datasets_path = refseq_dir + "ncbi_dataset.zip" attempts = 0 while attempts < max_attempts: if attempts: - eprint("\t\t\t\tReattempting", flush=True) - if os.path.isfile(rs_datasets_path): - os.remove(rs_datasets_path) + logger.info("Reattempting") + if Path(rs_datasets_path).is_file(): + Path(rs_datasets_path).unlink() attempts += 1 rs_datasets_cmd = run_datasets( None, acc_file_re, refseq_dir, True, api=api, verbose=True @@ -526,25 +494,23 @@ def prep_taxa_cols( try: with zipfile.ZipFile(rs_datasets_path, "r") as zip_ref: zip_ref.extractall(refseq_dir) - os.remove(rs_datasets_path) + Path(rs_datasets_path).unlink() break except zipfile.BadZipFile: - eprint( - f"\t\t\tERROR: datasets download corrupted - {attempts}", flush=True - ) + logger.error(f"datasets download corrupted - {attempts}") if attempts == max_attempts: sys.exit(10) except FileNotFoundError: - eprint(f"\t\tERROR: datasets failed - {attempts}", flush=True) + logger.error(f"datasets failed - {attempts}") if rs_datasets_cmd: - eprint(f"\t\tWARNING: datasets failed, assuming no genomes found", flush=True) + logger.warning(f"datasets failed, assuming no genomes found") acc2org_rs, acc2meta_rs = {}, {} else: acc2org_rs, acc2meta_rs, org_failed_2 = compile_organism_names( refseq_dir + "ncbi_dataset/" ) - print(f"\t\t{len(acc2meta_rs)} genome(s) queried from RefSeq", flush=True) + logger.debug(f"{len(acc2meta_rs)} genome(s) queried from RefSeq") acc2org, acc2meta = {**acc2org, **acc2org_n, **acc2org_rs}, { **acc2meta, @@ -602,7 +568,7 @@ def clean_ncbi_df(ncbi_df, update_path, kingdom="Fungi", api=None, max_attempts= acc2org_path = update_path + "../gca2org.tsv" acc2org = {} - if os.path.isfile(acc2org_path): + if Path(acc2org_path).is_file(): with open(acc2org_path, "r") as raw: for line in raw: d = line.split("\t") @@ -629,11 +595,10 @@ def clean_ncbi_df(ncbi_df, update_path, kingdom="Fungi", api=None, max_attempts= ncbi_df, update_path + "taxonomy/", api=api, acc2org=acc2org ) - with open(acc2org_path + ".tmp", "w") as out: + with atomic_write(acc2org_path) as out: for acc, org in acc2org.items(): org_meta = f'{org["genus"]}\t{org["species"]}\t{org["strain"]}' out.write(f"{acc}\t{org_meta}\n") - os.rename(acc2org_path + ".tmp", acc2org_path) # remove entries without sufficient metadata ncbi_df = ncbi_df.dropna(subset=["genus"]) @@ -763,8 +728,8 @@ def mk_wrk_dirs(update_path): """Make the download directories in the update path""" wrk_dirs = ["faa/", "fna/", "gff3/"] for wrk_dir in wrk_dirs: - if not os.path.isdir(update_path + wrk_dir): - os.mkdir(update_path + wrk_dir) + if not Path(update_path + wrk_dir).is_dir(): + Path(update_path + wrk_dir).mkdir() def prepare_ref_db(ref_db, date): @@ -780,9 +745,7 @@ def prepare_ref_db(ref_db, date): {k: v for k, v in ref_db.items() if v["source"].lower() == "ncbi"}, index="ome" ) if set(ref_db.keys()).difference(set(jgi.keys()).union(set(ncbi.keys()))): - eprint( - '\tWARNING: reference entries that are not labeled "jgi/ncbi" are excluded' - ) + logger.warning('\tWARNING: reference entries that are not labeled "jgi/ncbi" are excluded') return jgi.mtdb2pd(), ncbi.mtdb2pd() @@ -852,11 +815,9 @@ def internal_redundancy_check(db): def read_prev_tax(tax_path): """Open a genus to taxonomy JSON path""" tax_dicts = {} - if os.path.isfile(tax_path): - with open(tax_path, "r") as raw: - for line in raw: - data = line.rstrip().split("\t") - tax_dicts[data[0]] = json.loads(data[1]) + for line in _read_ledger(tax_path): + data = line.split("\t") + tax_dicts[data[0]] = json.loads(data[1]) return tax_dicts @@ -883,19 +844,19 @@ def ref_update( acquired external from any existing primary MTDB""" # NEED to mark none for new databases' refdb # initialize update - print("\nInitializing run", flush=True) + logger.info("Initializing run") mk_wrk_dirs(update_path) jgi_df, ncbi_df = prepare_ref_db(ref_db, date) # run JGI if jgi and len(jgi_df) > 0: - print("\nAssimilating MycoCosm", flush=True) + logger.info("Assimilating MycoCosm") jgi_db_path = update_path + date + ".jgi.mtdb" jgi_predb_path = update_path + date + ".jgi.predb2.mtdb" - if not os.path.isfile(jgi_predb_path): - print("\tDownloading MycoCosm data", flush=True) + if not Path(jgi_predb_path).is_file(): + logger.info("Downloading MycoCosm data") post_jgi_df, jgi_failed = jgiDwnld(jgi_df, update_path, jgi_email, jgi_pwd) jgi_predb = post_jgi_df.rename( columns={ @@ -905,7 +866,7 @@ def ref_update( } ) - print("\tCurating MycoCosm data", flush=True) + logger.info("Curating MycoCosm data") jgi_premtdb = jgi_predb.fillna("").to_dict(orient="list") jgi_mtdb, jgi_failed1 = predb2mtdb( jgi_premtdb, @@ -935,9 +896,9 @@ def ref_update( jgi_mtdb = mtdb() new_db = jgi_mtdb.mtdb2pd() - print("\nAssimilating NCBI", flush=True) - if not os.path.isfile(update_path + date + ".ncbi.predb"): - print("\tDownloading NCBI data", flush=True) + logger.info("Assimilating NCBI") + if not Path(update_path + date + ".ncbi.predb").is_file(): + logger.info("Downloading NCBI data") if ncbi_fallback: from mycotools.ncbi_dwnld_fallback import main as ncbi_dwnld_fallback @@ -984,8 +945,8 @@ def ref_update( # refdbncbi = mtdb(update_path + date + '.ncbi.ref.mtdb') ncbi_predb = pd.read_csv(update_path + date + ".ncbi.predb", sep="\t") - print("\tCurating NCBI data", flush=True) - if not os.path.isfile(update_path + date + ".ncbi.predb2.mtdb"): + logger.info("Curating NCBI data") + if not Path(update_path + date + ".ncbi.predb2.mtdb").is_file(): for key in ncbi_predb.columns: ncbi_predb[key] = ncbi_predb[key].fillna("") ncbi_predb["version"] = ncbi_predb["version"].astype(str) @@ -1019,7 +980,7 @@ def ref_update( # df2db(ncbi_db, ncbi_db_path) new_db = pd.concat([new_db, ncbi_db]) - print("\nAssimilating NCBI taxonomy data", flush=True) + logger.info("Assimilating NCBI taxonomy data") new_mtdb = mtdb.pd2mtdb(new_db) if kingdom.lower() == "fungi": @@ -1036,7 +997,7 @@ def ref_update( elif jgi_mtdb: update_mtdb = jgi_mtdb else: - eprint("\nNo updates", flush=True) + logger.info("No updates") sys.exit(0) if taxonomy: # already completed @@ -1126,10 +1087,10 @@ def taxonomy_update( taxless_db["taxonomy"] = [{} for x in taxless_db["taxonomy"]] tax_path = f"{update_path}../taxonomy.tsv" gca_path = f"{update_path}../gca2org.tsv" - if os.path.isfile(tax_path): - os.rename(tax_path, update_path + "old_taxonomy.tsv") - if os.path.isfile(gca_path): - os.rename(gca_path, update_path + "old_gca2org.tsv") + if Path(tax_path).is_file(): + Path(tax_path).rename(update_path + "old_taxonomy.tsv") + if Path(gca_path).is_file(): + Path(gca_path).rename(update_path + "old_gca2org.tsv") tax_dicts = gather_taxonomy( taxless_db, api_key=ncbi_api, king=group, rank=rank, output_path=tax_path ) @@ -1162,7 +1123,7 @@ def rogue_update( """Initialize/update a standalone primary MTDB""" # NEED to mark none for new databases' refdb # initialize update - print("\nInitializing run", flush=True) + logger.info("Initializing run") mk_wrk_dirs(update_path) prev_failed = parse_failed( rerun=rerun, file_path=format_path("$MYCODB/../log/failed.tsv") @@ -1180,7 +1141,7 @@ def rogue_update( ncbi_db_path = update_path + date + ".ncbi.mtdb" pre_ncbi_df0 = dwnld_ncbi_metadata(update_path + date + ".ncbi.tsv", group=group) pre_ncbi_df1 = pre_ncbi_df0.rename(columns={"Assembly Accession": "assembly_acc"}) - print("\tAcquiring NCBI metadata", flush=True) + logger.info("Acquiring NCBI metadata") ncbi_df, acc2meta = clean_ncbi_df( pre_ncbi_df1, update_path, kingdom=kingdom, api=ncbi_api ) @@ -1193,8 +1154,8 @@ def rogue_update( # if x not in {'genus', 'species', 'strain'})} if lineage_constraints: lineage_path = update_path + date + ".ncbi.posttax.df" - if not os.path.isfile(lineage_path): - print("\nExtracting lineages from NCBI", flush=True) + if not Path(lineage_path).is_file(): + logger.info("Extracting lineages from NCBI") # NEED to transition to datasets tax_dicts, ncbi_df = extract_constraint_lineages( ncbi_df, ncbi_api, kingdom, lineage_constraints, tax_dicts, tax_path @@ -1207,11 +1168,11 @@ def rogue_update( old_len = len(db["ome"]) new_len = len(db["ome"]) if old_len - new_len: - print("\t" + str(old_len - new_len) + " redundant entries removed", flush=True) + logger.debug("" + str(old_len - new_len) + " redundant entries removed") # run JGI if jgi: - print("\nAssimilating MycoCosm (1 download/minute)", flush=True) + logger.info("Assimilating MycoCosm (1 download/minute)") jgi_db_path = update_path + date + ".jgi.mtdb" mycocosm_path = update_path + date + ".mycocosm.csv" @@ -1223,8 +1184,8 @@ def rogue_update( # extract JGI lineages of interest and store tax_dicts for later if lineage_constraints: lineage_path = update_path + date + ".jgi.posttax.df" - if not os.path.isfile(lineage_path): - print("\tExtracting lineages from MycoCosm", flush=True) + if not Path(lineage_path).is_file(): + logger.info("Extracting lineages from MycoCosm") tax_dicts, jgi_df = extract_constraint_lineages( jgi_df, ncbi_api, kingdom, lineage_constraints, tax_dicts, tax_path ) @@ -1232,11 +1193,11 @@ def rogue_update( else: jgi_df = pd.read_csv(lineage_path, sep="\t") - print("\tSearching NCBI for MycoCosm overlap", flush=True) + logger.info("Searching NCBI for MycoCosm overlap") jgi_ncbi_overlap_file = f"{update_path}/redundant_ncbi.tsv" jgi2ncbi = parse_jgi2ncbi(update_path + "../jgi2ncbi.tsv") ncbi_df = ncbi_df.set_index("assembly_acc", drop=False) - if os.path.isfile(jgi_ncbi_overlap_file): + if Path(jgi_ncbi_overlap_file).is_file(): with open(jgi_ncbi_overlap_file, "r") as raw: todel_i = [x.rstrip() for x in raw] ncbi_df, ncbi_jgi_overlap = exec_rm_overlap(ncbi_df, todel_i) @@ -1246,10 +1207,9 @@ def rogue_update( rm_ncbi_overlap(ncbi_df, jgi_df, jgi2ncbi, true_ncbi, acc2meta, api=api) ) - print("\t\t" + str(len(jgi2ncbi)) + " overlapping genomes", flush=True) - with open(jgi_ncbi_overlap_file + ".tmp", "w") as out: + logger.debug("" + str(len(jgi2ncbi)) + " overlapping genomes") + with atomic_write(jgi_ncbi_overlap_file) as out: out.write("\n".join([x for x in todel_i])) - os.rename(jgi_ncbi_overlap_file + ".tmp", jgi_ncbi_overlap_file) add_true_ncbi(true_ncbi, update_path + "../supported_ncbi.tsv") add_jgi2ncbi(jgi2ncbi, update_path + "../jgi2ncbi.tsv") for i, row in jgi_df.iterrows(): @@ -1259,7 +1219,7 @@ def rogue_update( ): jgi_df.at[i, "biosample"] = jgi2biosample[row["portal"].lower()] - print("\tDownloading MycoCosm data", flush=True) + logger.info("Downloading MycoCosm data") jgi_predb_path = update_path + date + ".jgi.predb2.mtdb" jgi_predb, db, jgi_failed = jgi2db( jgi_df, @@ -1283,8 +1243,8 @@ def rogue_update( ncbi_df = pd.concat([ncbi_df, ncbi_jgi_overlap]) refdbjgi = mtdb.pd2mtdb(db) - if not os.path.isfile(jgi_predb_path): - print("\tCurating MycoCosm data", flush=True) + if not Path(jgi_predb_path).is_file(): + logger.info("Curating MycoCosm data") jgi_premtdb = jgi_predb.fillna("").to_dict(orient="list") if "assemblyPath" in jgi_premtdb: jgi_mtdb, jgi_failed1 = predb2mtdb( @@ -1318,7 +1278,7 @@ def rogue_update( jgi_db = pd.DataFrame({x: [] for x in refdbjgi.keys()}) new_db_path = update_path + date + ".checkpoint.jgi.mtdb" - if not os.path.isfile(new_db_path): + if not Path(new_db_path).is_file(): if len(jgi_db) > 0: df2db(jgi_db, jgi_db_path) if not db is None: @@ -1335,11 +1295,11 @@ def rogue_update( new_db = db new_dups = duplicates - print("\nAssimilating NCBI (10 download/second w/API key, 3 w/o)", flush=True) + logger.info("Assimilating NCBI (10 download/second w/API key, 3 w/o)") new_db["version"] = new_db["version"].astype(str) - if not os.path.isfile(update_path + date + ".ncbi.predb"): + if not Path(update_path + date + ".ncbi.predb").is_file(): # if not os.path.isfile(update_path + date + '.ncbi.predb'): - print("\tDownloading NCBI data", flush=True) + logger.info("Downloading NCBI data") ncbi_predb, new_db, ncbi_failed1 = ncbi2db( update_path, ncbi_df, @@ -1369,8 +1329,8 @@ def rogue_update( refdbncbi = mtdb(update_path + date + ".ncbi.ref.mtdb") ncbi_predb = pd.read_csv(update_path + date + ".ncbi.predb", sep="\t") - print("\tCurating NCBI data", flush=True) - if not os.path.isfile(update_path + date + ".ncbi.predb2.mtdb"): + logger.info("Curating NCBI data") + if not Path(update_path + date + ".ncbi.predb2.mtdb").is_file(): for key in ncbi_predb.columns: ncbi_predb[key] = ncbi_predb[key].fillna("") ncbi_predb["version"] = ncbi_predb["version"].astype(str) @@ -1410,7 +1370,7 @@ def rogue_update( new_mtdb = mtdb.pd2mtdb(new_db) - print("\nAssimilating NCBI taxonomy data", flush=True) + logger.info("Assimilating NCBI taxonomy data") if kingdom.lower() == "fungi": rank = "kingdom" else: @@ -1437,7 +1397,7 @@ def rogue_update( elif jgi_mtdb: update_mtdb = jgi_mtdb else: - eprint("\nNo updates", flush=True) + logger.info("No updates") sys.exit(0) return new_mtdb, update_mtdb @@ -1446,15 +1406,15 @@ def rogue_update( def rm_raw_data(out_dir): """Remove raw data after completion""" for i in ["faa", "gff3", "gff", "xml", "fna"]: - if os.path.isdir(out_dir + i): + if Path(out_dir + i).is_dir(): shutil.rmtree(out_dir + i) def gen_algn_db(update_path, omes): """Generate an alignment database for the complete primary MTDB""" - date = os.path.basename(os.path.abspath(update_path)) + date = Path(os.path.abspath(update_path)).name fas = collect_files(os.environ["MYCOFAA"] + "/", ".faa") - fas = [x for x in fas if os.path.basename(x)[:-6] in omes] + fas = [x for x in fas if Path(x).name[:-6] in omes] mkdb_base = "cat " + " ".join(fas) mkdb_blast = ( mkdb_base @@ -1476,13 +1436,11 @@ def gen_algn_db(update_path, omes): # with open(update_path + date + '_mmseqsdb.sh', 'w') as out: # out.write(mkdb_mmseqs) - print( - "\nOPTIONAL: To generate blastdb | mmseqsdb, run the following" + logger.debug("OPTIONAL: To generate blastdb | mmseqsdb, run the following" + "\nbash " + update_path + date - + "_makeblastdb.sh" - ) + + "_makeblastdb.sh") # bash ' + update_path \ # + date + '_mmseqsdb.sh') @@ -1552,7 +1510,7 @@ def check_add_mtdb(orig_mtdb, add_mtdb, update_path, overwrite=True): new_ome2old_ome = {v: k for k, v in old_ome2new_ome.items()} new_ome_mtdb = new_ome_mtdb.set_index("ome") for k, v in old_ome2new_ome.items(): - print(f"\t{k} converted to {v}", flush=True) + logger.debug(f"{k} converted to {v}") # create directories for new files fna_dir, gff_dir, faa_dir = ( @@ -1561,8 +1519,8 @@ def check_add_mtdb(orig_mtdb, add_mtdb, update_path, overwrite=True): f"{update_path}faa/", ) for path_ in [fna_dir, gff_dir, faa_dir]: - if not os.path.isdir(path_): - os.mkdir(path_) + if not Path(path_).is_dir(): + Path(path_).mkdir() # convert the file header names to the new omes for ome, old_ome in new_ome2old_ome.items(): @@ -1616,7 +1574,7 @@ def db2primary(addDB, refDB, save=False, combined=False): updates = {} refDB = refDB.set_index() if refOmes.intersection(addOmes) and not combined: - eprint(refOmes.intersection(addOmes), flush=True) + logger.info(refOmes.intersection(addOmes)) raise KeyError( "ERROR: ome codes exist in database. Rerun predb2mtdb or remove manually" ) @@ -1626,17 +1584,17 @@ def db2primary(addDB, refDB, save=False, combined=False): update_ome = base_ome2update_ome[base_ome] updates[update_ome] = ome del refDB[update_ome] - if os.path.isfile(addDB["gff3"][i]): + if Path(addDB["gff3"][i]).is_file(): move_ns(addDB["gff3"][i], format_path("$MYCOGFF3/" + ome + ".gff3")) - elif not os.path.isfile(format_path("$MYCOGFF3/" + ome + ".gff3")): + elif not Path(format_path("$MYCOGFF3/" + ome + ".gff3")).is_file(): raise FileNotFoundError(f"{ome} missing gff3 for unknown reason") - if os.path.isfile(addDB["fna"][i]): + if Path(addDB["fna"][i]).is_file(): move_ns(addDB["fna"][i], format_path("$MYCOFNA/" + ome + ".fna")) - elif not os.path.isfile(format_path("$MYCOFNA/" + ome + ".fna")): + elif not Path(format_path("$MYCOFNA/" + ome + ".fna")).is_file(): raise FileNotFoundError(f"{ome} missing fna for unknown reason") - if os.path.isfile(addDB["faa"][i]): + if Path(addDB["faa"][i]).is_file(): move_ns(addDB["faa"][i], format_path("$MYCOFAA/" + ome + ".faa")) - elif not os.path.isfile(format_path("$MYCOFAA/" + ome + ".faa")): + elif not Path(format_path("$MYCOFAA/" + ome + ".faa")).is_file(): raise FileNotFoundError(f"{ome} missing faa for unknown reason") addDB["gff3"][i] = os.environ["MYCOGFF3"] + ome + ".gff3" addDB["fna"][i] = os.environ["MYCOFNA"] + ome + ".fna" @@ -1683,37 +1641,35 @@ def control_flow( kingdom = kingdom.lower() if kingdom not in abbr2king: if kingdom not in set(abbr2king.values()): - eprint("\nERROR: invalid --kingdom", flush=True) + logger.error("invalid --kingdom") sys.exit(431) else: kingdom = abbr2king[kingdom] if not init and not update and not reference and not add and not taxonomy: - eprint( - "\nERROR: --update/--init/--reference/--add must be specified", flush=True - ) + logger.error("--update/--init/--reference/--add must be specified") sys.exit(15) elif reference and not init: - eprint("\nERROR: --reference requires a --init directory", flush=True) + logger.error("--reference requires a --init directory") sys.exit(14) elif lineage and not rank: - eprint("\nERROR: --lineage requires --rank") + logger.error("--lineage requires --rank") sys.exit(16) elif lineage and not init: - eprint("\nERROR: --lineage requires --init") + logger.error("--lineage requires --init") sys.exit(17) elif predb and not init: - eprint("\nERROR: --predb requires --init") + logger.error("--predb requires --init") sys.exit(18) elif predb and lineage: - eprint("\nERROR: --predb and --lineage are incompatible") + logger.error("--predb and --lineage are incompatible") sys.exit(20) elif reference: if add: - eprint("\nERROR: --add and --reference are incompatible") + logger.error("--add and --reference are incompatible") sys.exit(13) elif predb: - eprint("\nERROR: --reference and --predb are incompatible") + logger.error("--reference and --predb are incompatible") sys.exit(19) else: ref_db = mtdb(format_path(reference), add_paths=False) @@ -1735,11 +1691,11 @@ def control_flow( lineage_constraints = split_input(lineage) rank_constraints = split_input(rank) if len(lineage_constraints) != len(rank_constraints): - eprint("\nERROR: --lineage must be same length as --rank") + logger.error("--lineage must be same length as --rank") sys.exit(18) for rank_c in rank_constraints: if rank_c.lower() not in permitted_ranks: - eprint(f"\nERROR: accepted ranks: {permitted_ranks}") + logger.error(f"accepted ranks: {permitted_ranks}") sys.exit(22) rank2lineages = defaultdict(set) for i, v in enumerate(lineage_constraints): @@ -1753,14 +1709,14 @@ def control_flow( config = {} if "MYCODB" in os.environ: config_path = format_path("$MYCODB/../config/mtdb.json") - if os.path.isfile(config_path): + if Path(config_path).is_file(): config = read_json(format_path(config_path)) # for LEGACY installs: if "lineage_constraints" not in config: config["lineage_constraints"] = {} write_json(config, config_path) elif not init: - eprint("\nERROR: corrupted MycotoolsDB - no configuration found") + logger.error("corrupted MycotoolsDB - no configuration found") sys.exit(21) if not init: # is MYCODB initialized? # rogue_bool = config['rogue'] @@ -1769,13 +1725,11 @@ def control_flow( config["nonpublished"] = validate_t_and_c(config, discrepancy=True) write_json(config, config_path) if bool(config["jgi"]) and bool(ncbi_only): # and not overwrite: - eprint( - "\nERROR: --ncbi_only specified after initialization", flush=True - ) + logger.error("--ncbi_only specified after initialization") sys.exit(173) elif init: if format_path(init) != format_path(os.environ["MYCODB"] + "../../"): - eprint("\nERROR: MTDB linked. Unlink via `mtdb -u`") + logger.error("MTDB linked. Unlink via `mtdb -u`") sys.exit(175) # nonfungi is nonpublished by default because it is all GenBank @@ -1803,7 +1757,7 @@ def control_flow( if init: dbtype = kingdom init_dir = format_path(init) - if os.path.isdir(init_dir): + if Path(init_dir).is_dir(): init_dir += "mycotoolsdb/" if not init_dir.endswith("/"): init_dir += "/" @@ -1830,8 +1784,8 @@ def control_flow( os.environ[env] = envs[env] orig_db = db2df(mtdb()) # initialize a new database update_path = output + "log/" + date + "/" - if not os.path.isdir(update_path): - os.mkdir(update_path) + if not Path(update_path).is_dir(): + Path(update_path).mkdir() mtdb_initialize( init_dir, init=True ) # init_dir + 'config/mtdb.json', init = True) @@ -1839,14 +1793,14 @@ def control_flow( try: output = format_path("$MYCODB/..") except KeyError: - eprint("\nERROR: MTDB not linked. Link via `mtdb -i `", flush=True) + logger.error("MTDB not linked. Link via `mtdb -i `") sys.exit(50) update_path = output + "log/" + date + "/" - if not os.path.isdir(update_path): - os.mkdir(update_path) + if not Path(update_path).is_dir(): + Path(update_path).mkdir() if not True: # config['rogue']: # NEED TO MAKE THIS wget a particular URL old_db = db2df(db_path) - shutil.move(db_path, update_path + os.path.basename(db_path)) + shutil.move(db_path, update_path + Path(db_path).name) git_pull = subprocess.call( [ "git", @@ -1906,39 +1860,39 @@ def control_flow( ) if init_failed: if not failed: - eprint("\nERROR: some genomes failed curation", flush=True) + logger.error("some genomes failed curation") sys.exit(23) else: - eprint("\nWARNING: some genomes failed curation", flush=True) + logger.warning("some genomes failed curation") else: addDB = mtdb(format_path(add)) # we need full Paths for an addDB gff_fail, fna_fail, faa_fail = False, False, False - if not all(os.path.isfile(format_path(x)) for x in addDB.reset_index()["gff3"]): - eprint("\nERROR: some GFF paths do not exist", flush=True) + if not all(Path(format_path(x)).is_file() for x in addDB.reset_index()["gff3"]): + logger.error("some GFF paths do not exist") gff_fail = [ x for x in addDB.reset_index()["gff3"] - if not os.path.isfile(format_path(x)) + if not Path(format_path(x)).is_file() ] - print(",".join(gff_fail), flush=True) - if not all(os.path.isfile(format_path(x)) for x in addDB.reset_index()["fna"]): - eprint("\nERROR: some FNA paths do not exist", flush=True) + logger.debug(",".join(gff_fail)) + if not all(Path(format_path(x)).is_file() for x in addDB.reset_index()["fna"]): + logger.error("some FNA paths do not exist") fna_fail = [ x for x in addDB.reset_index()["fna"] - if not os.path.isfile(format_path(x)) + if not Path(format_path(x)).is_file() ] - print(",".join(fna_fail), flush=True) - if not all(os.path.isfile(format_path(x)) for x in addDB.reset_index()["faa"]): - eprint("\nERROR: some FAA paths do not exist", flush=True) + logger.debug(",".join(fna_fail)) + if not all(Path(format_path(x)).is_file() for x in addDB.reset_index()["faa"]): + logger.error("some FAA paths do not exist") faa_fail = [ x for x in addDB.reset_index()["faa"] - if not os.path.isfile(format_path(x)) + if not Path(format_path(x)).is_file() ] - print(",".join(faa_fail), flush=True) + logger.debug(",".join(faa_fail)) if gff_fail or fna_fail or faa_fail: sys.exit(124) @@ -1946,8 +1900,8 @@ def control_flow( # make date the acquisition time orig_mtdb = mtdb(primaryDB()) update_path = format_path("$MYCODB/../" + "log/" + date + "/") - if not os.path.isdir(update_path): - os.mkdir(update_path) + if not Path(update_path).is_dir(): + Path(update_path).mkdir() shutil.copy(primaryDB(), update_path) tax_path = f"{update_path}../taxonomy.tsv" @@ -1972,7 +1926,7 @@ def control_flow( if new_db_path != db_path: if db_path: - os.remove(db_path) + Path(db_path).unlink() return new_db_path if taxonomy: @@ -1991,10 +1945,7 @@ def control_flow( sys.exit(0) elif reference: if any(not x for x in ref_db["published"]) and not nonpublished: - eprint( - "\nWARNING: nonpublished data detected in reference and will be ignored", - flush=True, - ) + logger.warning("nonpublished data detected in reference and will be ignored") new_mtdb, update_mtdb = ref_update( ref_db, @@ -2037,13 +1988,13 @@ def control_flow( ) if not update_mtdb: - eprint("\nNo new data acquired", flush=True) + logger.info("No new data acquired") if not save: # add the predb2mtdb and remove files # df2db(db, format_path('$MYCODB/' + date + '.mtdb')) # output new database and new list of omes - eprint("\nMoving data into database", flush=True) + logger.info("Moving data into database") write_forbid_omes( set(new_mtdb["ome"]), format_path("$MYCODB/../log/relics.txt") ) @@ -2056,23 +2007,21 @@ def control_flow( ) full_mtdb.df2db(new_path + ".tmp") try: - shutil.move(primaryDB(), update_path + os.path.basename(primaryDB())) + shutil.move(primaryDB(), update_path + Path(primaryDB()).name) # move master database to log if it exists except FileNotFoundError: pass shutil.move(new_path + ".tmp", new_path) rm_raw_data(update_path) - eprint("\nMTDB update complete", flush=True) + logger.info("MTDB update complete") # gen_algn_db( # update_path, set(full_mtdb['ome']) # ) else: # NEED to: insert note aboutrunning updatedb on predb new_mtdb.df2db(format_path(update_path + date + ".mtdb")) - eprint( - f"\nUpdate ready for `mtdb u -a` at " - + f'{format_path(update_path + date + ".mtdb")}' - ) + logger.info(f"Update ready for `mtdb u -a` at " + + f'{format_path(update_path + date + ".mtdb")}') # output new database and new list of omes return primaryDB() @@ -2175,6 +2124,7 @@ def main(): ) run_args.add_argument("-c", "--cpu", type=int, default=1) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) args_dict = { "Primary MTDB": primaryDB(verbose=False), diff --git a/mycotools/utils/curGFF3.py b/mycotools/utils/curGFF3.py index eee6a52..7adbdba 100755 --- a/mycotools/utils/curGFF3.py +++ b/mycotools/utils/curGFF3.py @@ -16,15 +16,17 @@ # RNAs with no parent # CDS/exons with no parent -import os +import logging import re import sys import copy from collections import defaultdict from itertools import combinations, chain -from mycotools.lib.kontools import format_path, sys_start, eprint +from mycotools.lib.kontools import format_path, sys_start from mycotools.lib.biotools import gff2list, list2gff, gff3Comps +logger = logging.getLogger(__name__) + class RNAError(Exception): pass @@ -1018,7 +1020,7 @@ def main(gff_path, ome, cur_seqids=False): typ = True if not typ: - eprint("\tERROR: type unknown ", flush=True) + logger.error("type unknown ") return None new_gff = curGff3(gff, ome, cur_seqids) diff --git a/mycotools/utils/extractHmmAcc.py b/mycotools/utils/extractHmmAcc.py index dcc0bfd..3c5ca08 100755 --- a/mycotools/utils/extractHmmAcc.py +++ b/mycotools/utils/extractHmmAcc.py @@ -1,9 +1,12 @@ #! /usr/bin/env python3 -import os +import logging import re import sys -from mycotools.lib.kontools import eprint, file2list +from mycotools.lib.kontools import file2list +from pathlib import Path + +logger = logging.getLogger(__name__) def grabAccs(db_str): @@ -35,10 +38,7 @@ def hmmExtract(accession, db_str): r"HMMER\d\/f \[.*?\]\nNAME +" + accession + r"[\s\S]*?\/\/", db_str ) if not hmm_search: - eprint( - "\nERROR: " + accession + " does not exist or unexpected error\n", - flush=True, - ) + logger.error("" + accession + " does not exist or unexpected error\n") sys.exit(2) hmm = hmm_search[0] + "\n" @@ -49,7 +49,7 @@ def hmmExtract(accession, db_str): def main(hmm_db, accessions=False): if accessions: - if os.path.isfile(accessions): + if Path(accessions).is_file(): accessions = file2list(accessions) hmm_str = "" for accession in accessions: @@ -80,22 +80,22 @@ def cli(): print(usage, flush=True) sys.exit(1) - if not os.path.isfile(sys.argv[1]): - eprint("\nERROR: Invalid `.hmm` database path", flush=True) + if not Path(sys.argv[1]).is_file(): + logger.error("Invalid `.hmm` database path") sys.exit(2) - print("\nReading hmm database ...", flush=True) + logger.info("Reading hmm database ...") with open(sys.argv[1], "r") as raw_hmm_db: hmm_db = raw_hmm_db.read() if len(sys.argv) > 2: hmm_str = main(hmm_db, accessions=sys.argv[2]) - print("\nWriting abstracted hmms ...", flush=True) + logger.info("Writing abstracted hmms ...") with open(sys.argv[2] + ".hmm", "w") as out: out.write(hmm_str) else: - if not os.path.isdir("hmm"): - os.mkdir("hmm") + if not Path("hmm").is_dir(): + Path("hmm").mkdir() hmm_strs = main(hmm_db) for accession in hmm_strs: with open("hmm/" + accession + ".hmm", "w") as out: diff --git a/mycotools/utils/extractHmmsearch.py b/mycotools/utils/extractHmmsearch.py index 7cc6589..0810e42 100755 --- a/mycotools/utils/extractHmmsearch.py +++ b/mycotools/utils/extractHmmsearch.py @@ -1,10 +1,13 @@ #! /usr/bin/env python3 -import os +import logging import re import sys import argparse -from mycotools.lib.kontools import intro, outro, file2list, format_path, mkOutput +from mycotools.lib.kontools import intro, outro, file2list, format_path, mkOutput, setup_logging +from pathlib import Path + +logger = logging.getLogger(__name__) def grab_names(data, query=False): @@ -98,10 +101,7 @@ def grab_hits( if x != "" and x != ".." and "[" not in x and "]" not in x ] if len(outLine) != 13: - print( - "\nINVALID ALIGNMENT HEADERS - CHECK SCRIPT/ALIGNMENT\n", - flush=True, - ) + logger.error("INVALID ALIGNMENT HEADERS - CHECK SCRIPT/ALIGNMENT") sys.exit(5) if threshold: hmmStart = int(outLine[6]) @@ -196,7 +196,7 @@ def grab_hits( else: hit_str, aln_str = None, None - print("did not work", flush=True) + logger.error("did not work") return hit_str, aln_str @@ -327,13 +327,14 @@ def cli(): "-o", "--output", help="Output file name/path (extensions automatically applied" ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) # initialize output file structure output = format_path(args.output) if not args.output: - output = mkOutput(os.getcwd() + "/", "extractHmmsearch") - elif not os.path.isdir(output): - os.mkdir(args.output) + output = mkOutput(str(Path.cwd()) + "/", "extractHmmsearch") + elif not Path(output).is_dir(): + Path(args.output).mkdir() args_dict = { "Input": args.input, @@ -352,24 +353,24 @@ def cli(): args.accession = True else: if args.query: - if os.path.isfile(args.query): + if Path(args.query).is_file(): ques = file2list(args.query) else: ques = [args.query] args.query = True elif args.accession: - if os.path.isfile(args.accession): + if Path(args.accession).is_file(): ques = file2list(args.accession) else: ques = [args.accession] args.accession = True else: - print("\nNeed `-q` or `-a` specified\n", flush=True) + logger.error("Need `-q` or `-a` specified") sys.exit(8) - if not os.path.isfile(args.input): - print("\n\tNot a valid input file\n", flush=True) - sys.eixt(2) + if not Path(args.input).is_file(): + logger.error("Not a valid input file") + sys.exit(2) with open(args.input, "r") as raw: data = raw.read() diff --git a/mycotools/utils/gff2gff3.py b/mycotools/utils/gff2gff3.py index 6ffa950..910a275 100755 --- a/mycotools/utils/gff2gff3.py +++ b/mycotools/utils/gff2gff3.py @@ -2,16 +2,19 @@ # NEED to arrive at a consensus for protein and transcript IDs +import logging import re import sys import copy import argparse from collections import defaultdict from mycotools.lib.biotools import gff2list, list2gff, gff2Comps, gff3Comps -from mycotools.lib.kontools import format_path, eprint, vprint +from mycotools.lib.kontools import format_path, setup_logging from mycotools.utils.gtf2gff3 import add_genes, remove_start_stop from mycotools.utils.curGFF3 import rename_and_organize +logger = logging.getLogger(__name__) + def gff2gff3(gff_list, ome, jgi_ome): @@ -236,14 +239,9 @@ def main(gff_list, ome, jgi_ome, safe=True, verbose=True): gff_list, safe=safe, comps=comps, gene_prefix=gene_prefix ) if failed: - vprint(str(len(failed)) + "\tgenes failed", v=verbose, e=True, flush=True) + logger.debug(str(len(failed)) + "genes failed") if flagged: - vprint( - str(len(flagged)) + "\tgene coordinates from exons", - v=verbose, - e=True, - flush=True, - ) + logger.debug(str(len(flagged)) + "gene coordinates from exons") pregff3 = gff2gff3(gff_prep, ome, jgi_ome) gff3 = resolve_alternate_splicing(pregff3) gff3 = rename_and_organize(gff3) @@ -253,11 +251,11 @@ def main(gff_list, ome, jgi_ome, safe=True, verbose=True): err_name.append(err.upper()) if verbose: if err == "ob": - eprint("ERROR: genes with out of bounds coordinates", flush=True) - eprint(",".join(err_list), flush=True) + logger.error("genes with out of bounds coordinates") + logger.info(",".join(err_list)) elif err == "nr": - eprint("ERROR: missing RNA", flush=True) - eprint(",".join(err_list), flush=True) + logger.error("missing RNA") + logger.info(",".join(err_list)) return gff3, errors @@ -275,9 +273,10 @@ def cli(): help="Fail genes w/o CDS sequences that lack start or stop codons", ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) gff_list = gff2list(format_path(args.input)) - eprint(args.ome + "\t" + args.input, flush=True) + logger.info(args.ome + "" + args.input) gff3, errors = main(gff_list, args.ome, args.jgi, args.fail) print(list2gff(gff3), flush=True) diff --git a/mycotools/utils/gtf2gff3.py b/mycotools/utils/gtf2gff3.py index 7e381c6..c7d8d2b 100755 --- a/mycotools/utils/gtf2gff3.py +++ b/mycotools/utils/gtf2gff3.py @@ -4,7 +4,7 @@ # NEED to arrive at a consensus for protein IDs # NEED to name tRNAs, pseudogenes, etc. similar to curGFF.py -import os +import logging import re import sys import copy @@ -20,9 +20,12 @@ gff3Comps, gff2Comps, ) -from mycotools.lib.kontools import collect_files, eprint, format_path +from mycotools.lib.kontools import collect_files, format_path, setup_logging from mycotools.gff2seq import aamain as gff2proteome from mycotools.utils.curGFF3 import rename_and_organize +from pathlib import Path + +logger = logging.getLogger(__name__) def grabOutput(output_pref): @@ -34,16 +37,16 @@ def grabOutput(output_pref): both. """ - output_path, pref = os.path.dirname(output_pref), os.path.basename(output_pref) + output_path, pref = str(Path(output_pref).parent), Path(output_pref).name output_files = collect_files(output_path, "*") - hits = [x for x in output_files if os.path.basename(x).startswith(pref)] + hits = [x for x in output_files if Path(x).name.startswith(pref)] proteome = [x for x in hits if x.endswith(".results.aa.fasta")] gtf = [x for x in hits if x.endswith(".results.gtf")] if len(gtf) != 1 or len(proteome) != 1: if len(gtf) < 1 or len(proteome) < 1: - print("\nERROR: complete output files not detected", flush=True) + logger.error("complete output files not detected") else: - print("\nERROR: multiple eligible complete output detected", flush=True) + logger.error("multiple eligible complete output detected") sys.exit(3) return gff2list(gtf[0]), fa2dict(proteome[0]) @@ -357,7 +360,7 @@ def conservativeRemoval(gene_dict_prep): ] flagged.append(gene) except IndexError: - eprint(gene + " cannot create gene coordinates", flush=True) + logger.info(gene + " cannot create gene coordinates") continue gene_dict[gene] = temp @@ -789,11 +792,12 @@ def cli(): # parser.add_argument('--fail', default = True, action = 'store_false', # help = 'Fail genes without start or stop codons.') args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) if args.output: output = format_path(args.output) - if not os.path.isdir(output): - os.mkdir(output) + if not Path(output).is_dir(): + Path(output).mkdir() output += "/" output += args.prefix else: @@ -810,7 +814,7 @@ def cli(): if args.prefix: if "_" in args.prefix: - eprint('\nERROR: "_" not allowed in prefix\n', flush=True) + logger.error('\nERROR: "_" not allowed in prefix\n') sys.exit(1) gff, trans_str, failed, flagged = main( @@ -827,11 +831,11 @@ def cli(): with open(output + ".transitions", "w") as out: out.write(trans_str) if failed: - print("\n" + str(len(failed)) + " failures\n", flush=True) + logger.debug("" + str(len(failed)) + " failures\n") with open(output + ".failed", "w") as out: out.write("\n".join(["\t".join(x) for x in failed])) if flagged: - print("\n" + str(len(flagged)) + " flagged\n", flush=True) + logger.debug("" + str(len(flagged)) + " flagged\n") with open(output + ".flagged", "w") as out: out.write("\n".join(flagged)) diff --git a/mycotools/utils/jgi2db.py b/mycotools/utils/jgi2db.py index 1addbfe..9390c59 100755 --- a/mycotools/utils/jgi2db.py +++ b/mycotools/utils/jgi2db.py @@ -1,5 +1,6 @@ #! /usr/bin/env python3 +import logging import os import re import sys @@ -13,17 +14,20 @@ import numpy as np from io import StringIO from mycotools.predb2mtdb import main as predb2mtdb -from mycotools.lib.kontools import intro, outro, eprint +from mycotools.lib.kontools import intro, outro, setup_logging from mycotools.lib.dbtools import db2df, df2db, readLog, log_editor from mycotools.jgiDwnld import jgi_login as jgi_login from mycotools.jgiDwnld import retrieve_xml as retrieve_xml from mycotools.jgiDwnld import jgi_dwnld as jgi_dwnld +from pathlib import Path + +logger = logging.getLogger(__name__) def compileLog(log_path): log = {} - if not os.path.isfile(log_path): + if not Path(log_path).is_file(): with open(log_path, "w") as out: out.write("#assembly_acc\tfna\tgff3\tfaa") else: @@ -158,22 +162,22 @@ def runjgi_dwnld( ran_dwnld = False if ome not in ome_set: - print(spacer + "\t" + ome + ": " + jgi_df["name"][i], flush=True) + logger.debug(spacer + "" + ome + ": " + jgi_df["name"][i]) for typ in dwnlds: if log[ome][typ] == "error": if not rerun: - print(spacer + "\t\t" + typ + ": ERROR", flush=True) + logger.debug(spacer + "" + typ + ": ERROR") continue check, preexisting, new_typ, ran_dwnld, org_name = jgi_dwnld( ome, typ, output, masked=masked ) if not isinstance(check, int): jgi_df.at[i, new_typ + "_path"] = check - base_check = os.path.basename(os.path.abspath(check)) - print(spacer + "\t\t" + new_typ + ": " + str(base_check), flush=True) + base_check = Path(os.path.abspath(check)).name + logger.debug(spacer + "" + new_typ + ": " + str(base_check)) log[ome][typ] = base_check else: - print(spacer + "\t\t" + new_typ + ": ERROR", flush=True) + logger.debug(spacer + "" + new_typ + ": ERROR") log[ome][typ] = "error" if typ in {"gff3", "fna"}: log_editor( @@ -257,7 +261,7 @@ def main( ome_col = "portal" name_col = "name" else: - print(spacer + "ERROR: invalid MycoCosm tsv headers", flush=True) + logger.debug(spacer + "invalid MycoCosm tsv headers") sys.exit(3) toDel = [] @@ -281,7 +285,7 @@ def main( jgi_df = jgi_df.drop(failed) jgi_df = jgi_df.reset_index() - print(spacer + "Redundancy check", flush=True) + logger.debug(spacer + "Redundancy check") if isinstance(ref_db, pd.DataFrame): ref_db["index"] = ref_db["assembly_acc"].copy() ref_db = ref_db.set_index("index") @@ -298,17 +302,17 @@ def main( else: updates.to_csv(f"{output}/jgiUpdates.tsv", sep="\t") # update_check = {i[-1]: i[0:3] for i in updates if i[0]} - print(spacer + "\t" + str(len(jgi_df)) + " genomes to assimilate", flush=True) + logger.debug(spacer + "" + str(len(jgi_df)) + " genomes to assimilate") else: new_ref_db, updates = None, {} - print(spacer + "Logging into JGI", flush=True) + logger.debug(spacer + "Logging into JGI") jgi_login(user, pwd) - if not os.path.exists(output + "/xml"): - os.mkdir(output + "/xml") + if not Path(output + "/xml").exists(): + Path(output + "/xml").mkdir() - print(spacer + "Retrieving `xml` directories", flush=True) + logger.debug(spacer + "Retrieving `xml` directories") ome_set, failed, count = set(), [], 0 for i, row in jgi_df.iterrows(): error_check, attempt = True, 0 @@ -323,7 +327,7 @@ def main( elif error_check != -1: time.sleep(0.3) if error_check != -1: - eprint(f"{spacer}\t{row[ome_col]} failed to retrieve XML", flush=True) + logger.info(f"{spacer}\t{row[ome_col]} failed to retrieve XML") ome_set.add(row[ome_col]) log_path = output + "/jgi2db.log" @@ -339,7 +343,7 @@ def main( jgi_df = jgi_df.drop(drop_index) ome_set.add(ome) - print(spacer + "Downloading JGI data", flush=True) + logger.debug(spacer + "Downloading JGI data") dwnlds = [] if assembly: dwnlds.append("fna") @@ -349,14 +353,14 @@ def main( dwnlds.append("faa") for typ in dwnlds: - if not os.path.isdir(output + "/" + typ): - os.mkdir(output + "/" + typ) + if not Path(output + "/" + typ).is_dir(): + Path(output + "/" + typ).mkdir() if typ == "gff3": - if not os.path.isdir(output + "/gff3"): - os.mkdir(output + "/gff3") + if not Path(output + "/gff3").is_dir(): + Path(output + "/gff3").mkdir() if all(x in log for x in list(jgi_df[ome_col])) and not rerun: - print(spacer + "\tAll downloaded, rerun off", flush=True) + logger.debug(spacer + "All downloaded, rerun off") jgi_df = jgi_df.set_index(ome_col) jgi_df = log2df(jgi_df, log, output) jgi_df = jgi_df.reset_index() @@ -384,10 +388,10 @@ def main( spacer, ) - if os.path.exists("cookies"): - os.remove("cookies") - if os.path.exists(os.path.expanduser("~/.nullJGIdwnld")): - os.remove(os.path.expanduser("~/.nullJGIdwnld")) + if Path("cookies").exists(): + Path("cookies").unlink() + if Path(str(Path("~/.nullJGIdwnld").expanduser())).exists(): + Path(str(Path("~/.nullJGIdwnld").expanduser())).unlink() jgi_df = jgi_df.rename( columns={ @@ -486,6 +490,7 @@ def cli(): ) args = parser.parse_args() + setup_logging(verbose=getattr(args, "verbose", False)) args_dict = { "Preexisting db": args.database, @@ -501,8 +506,8 @@ def cli(): output = os.path.abspath(args.output) else: output = start_time.strftime("%Y%m%d") + "_jgi2db" - if not os.path.isdir(output): - os.mkdir(output) + if not Path(output).is_dir(): + Path(output).mkdir() if args.login: with open(args.login, "r") as raw: @@ -520,12 +525,10 @@ def cli(): jgi_df = main(args.mycocosm, refdb, output) df2db(jgi_df, output + "/new.db") - print( - "\nSuccess! " - + str(len(jgi_df), flush=True) + logger.debug("Success! " + + str(len(jgi_df)) + " added to database\n \ - Run updateDB to confirm and finish update." - ) + Run updateDB to confirm and finish update.") outro(start_time) diff --git a/mycotools/utils/ncbi2db.py b/mycotools/utils/ncbi2db.py index bcaecec..5c84ac4 100755 --- a/mycotools/utils/ncbi2db.py +++ b/mycotools/utils/ncbi2db.py @@ -7,6 +7,7 @@ # NEED TO EDIT REDUNDANCY CHECK TO REFERENCE QUERIED ASSEMBLY ACCESSIONS FROM # BIOSAMPLES +import logging import os import re import sys @@ -16,11 +17,13 @@ import numpy as np import pandas as pd from datetime import datetime -from mycotools.lib.kontools import intro, outro, eprint +from mycotools.lib.kontools import intro, outro from mycotools.lib.dbtools import db2df, df2db, primaryDB from mycotools.ncbiDwnld import main as ncbi_dwnld from mycotools.predb2mtdb import main as predb2mtdb +logger = logging.getLogger(__name__) + def redundancy_check(db, ncbi_df, ass_acc, duplicates={}): """should expect that ncbi_df (pd.DataFrame()) is only comprised of biosamples that are @@ -157,7 +160,7 @@ def main( ncbi_df = ncbi_df.drop(i) ncbi_df = ncbi_df.reset_index() - print(spacer + "Redundancy check", flush=True) + logger.debug(spacer + "Redundancy check") update_check = {} if ref_db is not None: if len(ref_db) > 0: @@ -174,12 +177,10 @@ def main( update_check = {i[-2]: i for i in updates if i[0]} # dict(update_check) = {assembly_accNEW: [organism, ref organism, # old_assembly_acc]} - print( - spacer + "\t" + str(len(ncbi_df)) + " genomes to assimilate", flush=True - ) + logger.debug(spacer + "" + str(len(ncbi_df)) + " genomes to assimilate") if len(ncbi_df) > 0: - print(spacer + "Initializing NCBI acquisition", flush=True) + logger.debug(spacer + "Initializing NCBI acquisition") if fallback: from mycotools.ncbi_dwnld_fallback import main as ncbi_dwnld_fallback @@ -210,10 +211,7 @@ def main( spacer="\t\t\t", ) - print( - spacer + "\t" + str(len(ncbi_df)) + " entries with assemblies and gffs", - flush=True, - ) + logger.debug(spacer + "" + str(len(ncbi_df)) + " entries with assemblies and gffs") ncbi_df = ncbi_df.rename( columns={ "Release Date": "published", diff --git a/mycotools/utils/og2mycodb.py b/mycotools/utils/og2mycodb.py index f2058e3..8f5e647 100755 --- a/mycotools/utils/og2mycodb.py +++ b/mycotools/utils/og2mycodb.py @@ -1,11 +1,11 @@ #! /usr/bin/env python3 -import os import re import sys import multiprocessing as mp from mycotools.lib.biotools import gff2list, list2gff, gff3Comps from mycotools.lib.kontools import format_path, sys_start +from pathlib import Path def og2dict(orthogroup_file): @@ -107,7 +107,7 @@ def extract_ogs(ogInfo_dict, ogtag): def og2mycoDB(ogInfo_dict, omes=set(), file_path=format_path("$MYCOGFF3/../ogs.tsv")): out_list = [] - if os.path.isfile(file_path): + if Path(file_path).is_file(): with open(file_path, "r") as raw: for line in raw: data = line.rstrip().split("\t") diff --git a/pyproject.toml b/pyproject.toml index f3ed868..42da4f2 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,14 +4,30 @@ build-backend = "setuptools.build_meta" [project] name = "mycotools" -version = "1.0.0" +version = "2.0.0" authors = [{name="Zachary Konkel", email="konkelzach@protonmail.com"}] description = "Comparative genomics automation and standardization software" readme = "README.md" license = {file = "LICENSE"} -requires-python = '>=3.0,<4' -dependencies = ['biopython', 'pandas', 'requests', 'scipy', 'openpyxl', 'tqdm', - 'cryptography', 'ete3', 'pyqt5'] +requires-python = ">=3.9,<4" +# Versions are lower-bounded to the reference `mycotools` micromamba env +# (Python 3.9.21). numpy, clipkit, cogent3, and dna_features_viewer were +# previously undeclared despite being imported by the package. +dependencies = [ + "biopython>=1.85", + "pandas>=2.2.3", + "numpy>=2.0.2", + "scipy>=1.13.1", + "requests>=2.32.3", + "openpyxl>=3.1.5", + "tqdm>=4.67.1", + "cryptography>=44.0.0", + "ete3>=3.1.3", + "pyqt5>=5.15.9", + "clipkit>=2.4.1", + "cogent3>=2025.3.22a2", + "dna_features_viewer>=3.1.5", +] classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", From 279fa52c5d2211100ed7fb1d98676b14043a4420 Mon Sep 17 00:00:00 2001 From: xonq Date: Sat, 18 Jul 2026 02:47:10 +0000 Subject: [PATCH 02/34] remove usearch --- mycotools/fa2clus.py | 62 -------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/mycotools/fa2clus.py b/mycotools/fa2clus.py index 4c6748a..5de24a9 100755 --- a/mycotools/fa2clus.py +++ b/mycotools/fa2clus.py @@ -245,68 +245,6 @@ def rd_dmnd_distmtx(outputFile, minVal, pid=True): return distance_matrix # , outMatrix -def runUsearch(fasta, output, clus_var, cpus=1, verbose=False): - - if verbose: - subprocess.call( - [ - "usearch", - "-calc_distmx", - fasta, - "-tabbedout", - output, - "-clus_var", - clus_var, - "-threads", - str(cpus), - ] # stdout = subprocess.PIPE, - # stderr = subprocess.PIPE - ) - else: - subprocess.call( - [ - "usearch", - "-calc_distmx", - fasta, - "-tabbedout", - output, - "-clus_var", - clus_var, - "-threads", - str(cpus), - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - -def rd_usrch_distmtx(dis_path, sep="\t"): - """Imports a distance matrix with each line formatted as `organism $SEP organism $SEP distance`. - - this is equivalent to the `-tabbedout` argument in `usearch -calc_distmx`. The function - compiles a dictionary with the information and reciprocal information for each organism in each - line, then converts this dictionary of dictionaries into a pandas dataframe. As a distance matrix, - NA values are converted to maximum distance (1).""" - - distance_matrix = pd.DataFrame() - with open(dis_path, "r") as raw: - data = raw.read() - prepData = [x.rstrip() for x in data.split("\n") if x != ""] - - dist_dict = {} - for line in prepData: - vals = line.split(sep) - if vals[0] not in dist_dict: - dist_dict[vals[0]] = {} - if vals[1] not in dist_dict: - dist_dict[vals[1]] = {} - dist_dict[vals[0]][vals[1]] = float(vals[2]) - dist_dict[vals[1]][vals[0]] = float(vals[2]) - - distance_matrix = pd.DataFrame(dist_dict).sort_index(0).sort_index(1) - - return distance_matrix.fillna(1.0) - - def scikitaggd(distance_matrix, maxDist=0.6, linkage="single"): """Performs agglomerative clustering and extracts the cluster labels, then sorts according to cluster number. In the future, this will also extract a Newick tree.""" From db3bf4a897a25563509c2b05772f0f92f5932ac7 Mon Sep 17 00:00:00 2001 From: xonq Date: Sat, 18 Jul 2026 05:51:29 +0000 Subject: [PATCH 03/34] init testing --- mycotools/acc2fq.py | 2 - mycotools/acc2gbk.py | 4 - mycotools/add2gff.py | 2 +- mycotools/annotationStats.py | 1 - mycotools/assemblyStats.py | 1 - mycotools/coords2fa.py | 1 - mycotools/crap.py | 8 +- mycotools/db2files.py | 11 --- mycotools/db2hgs.py | 5 +- mycotools/db2microsyntree.py | 4 +- mycotools/db2search.py | 11 +-- mycotools/fa2clus.py | 4 - mycotools/fa2mass.py | 1 - mycotools/fa2tree.py | 3 - mycotools/gff2seq.py | 3 +- mycotools/jgiDwnld.py | 1 - mycotools/lib/biotools.py | 1 - mycotools/lib/dbtools.py | 6 +- mycotools/manage_mtdb.py | 2 +- mycotools/ncbiDwnld.py | 9 +-- mycotools/ncbi_dwnld_fallback.py | 10 +-- mycotools/predb2mtdb.py | 4 +- mycotools/treetools.py | 2 +- mycotools/update_mtdb.py | 10 --- mycotools/utils/curGFF3.py | 1 - mycotools/utils/gff2gff3.py | 3 +- mycotools/utils/gtf2gff3.py | 1 - mycotools/utils/jgi2db.py | 9 --- mycotools/utils/ncbi2db.py | 8 -- test/unit/README.md | 51 ++++++++++++ test/unit/_template.py | 63 +++++++++++++++ test/unit/conftest.py | 80 ++++++++++++++++++ test/unit/test_cli_smoke.py | 135 +++++++++++++++++++++++++++++++ test/unit/test_lib_biotools.py | 54 +++++++++++++ 34 files changed, 399 insertions(+), 112 deletions(-) create mode 100644 test/unit/README.md create mode 100644 test/unit/_template.py create mode 100644 test/unit/conftest.py create mode 100644 test/unit/test_cli_smoke.py create mode 100644 test/unit/test_lib_biotools.py diff --git a/mycotools/acc2fq.py b/mycotools/acc2fq.py index 4e06fe6..c8c3b51 100755 --- a/mycotools/acc2fq.py +++ b/mycotools/acc2fq.py @@ -5,8 +5,6 @@ import sys import gzip import argparse -from Bio import SeqIO -from collections import defaultdict # from mycotools.lib.biotools import dict2fq from mycotools.lib.kontools import format_path, stdin2str, setup_logging diff --git a/mycotools/acc2gbk.py b/mycotools/acc2gbk.py index 0345863..6486814 100755 --- a/mycotools/acc2gbk.py +++ b/mycotools/acc2gbk.py @@ -4,7 +4,6 @@ import re import sys import argparse -import multiprocessing as mp from itertools import chain from collections import defaultdict from mycotools.lib.kontools import format_path, stdin2str, setup_logging @@ -91,7 +90,6 @@ def contig2gbk( + "+" ) name = ome + "_" + contig - relative_end = seq_coords[-1][1] - seq_coords[0][0] gbk = ( "LOCUS " + name @@ -379,8 +377,6 @@ def gen_gbk( if count0: if v1[0] - v0[1] > break_contigs: breaks.add(k0) - k0 = k1 - v0 = v1 count0 += 1 count1 = 0 diff --git a/mycotools/add2gff.py b/mycotools/add2gff.py index 0be104d..4bc3ddd 100755 --- a/mycotools/add2gff.py +++ b/mycotools/add2gff.py @@ -12,7 +12,7 @@ import sys import argparse from collections import defaultdict -from mycotools.lib.kontools import sys_start, format_path, mkOutput, setup_logging +from mycotools.lib.kontools import format_path, mkOutput, setup_logging from mycotools.lib.biotools import gff2list, list2gff, gff3Comps, gff2Comps, gtfComps from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.utils.curGFF3 import rename_and_organize diff --git a/mycotools/annotationStats.py b/mycotools/annotationStats.py index 0e16f3f..3bd24ad 100755 --- a/mycotools/annotationStats.py +++ b/mycotools/annotationStats.py @@ -305,7 +305,6 @@ def main(in_path, log_path=None, cpus=1, db=None): def cli(): setup_logging() - output = False usage = "\nUSAGE: `gff`/`gtf`/`gff3` OR mycotoolsDB, optional output file\n" if "-h " in sys.argv or "--help" in sys.argv or "-h" == sys.argv[-1]: print(usage, flush=True) diff --git a/mycotools/assemblyStats.py b/mycotools/assemblyStats.py index 385d016..5fcc90e 100755 --- a/mycotools/assemblyStats.py +++ b/mycotools/assemblyStats.py @@ -9,7 +9,6 @@ import os import sys -import copy import logging import multiprocessing as mp from mycotools.lib.dbtools import mtdb diff --git a/mycotools/coords2fa.py b/mycotools/coords2fa.py index 40d7448..7e4578b 100755 --- a/mycotools/coords2fa.py +++ b/mycotools/coords2fa.py @@ -4,7 +4,6 @@ import logging import sys -import argparse from Bio.Seq import Seq from collections import defaultdict from mycotools.lib.biotools import fa2dict, dict2fa diff --git a/mycotools/crap.py b/mycotools/crap.py index cffa286..8d302d6 100755 --- a/mycotools/crap.py +++ b/mycotools/crap.py @@ -29,7 +29,7 @@ from collections import Counter, defaultdict try: - from ete3 import Tree, faces, TreeStyle, NodeStyle, AttrFace + from ete3 import Tree, faces, TreeStyle, NodeStyle from ete3.parser.newick import NewickError except ImportError: raise ImportError( @@ -350,7 +350,6 @@ def outgroup_mngr( fa2clus_log = read_json(log_path) algorithm = fa2clus_log["algorithm"] # use previous search algorithm successes = fa2clus_log["successes"] - iterations = fa2clus_log["iterations"] prev_size = len(fa2dict(clus_dir + "../" + str(focal_gene) + ".fa")) max_success = successes[0] @@ -972,7 +971,6 @@ def parse_log(log_path, new_log, out_dir): except FileNotFoundError: old_log = None - rereun_search = False if old_log: try: with open(old_log["db_path"], "rb") as raw: @@ -1527,7 +1525,6 @@ def hg_main( interval=interval, verbose=False, ) - out_query = query + ".outgroup" query_hits = all_keys out_keys = list(set(all_keys).difference(in_keys)) logger.debug("" + str(len(in_keys)) + " gene ingroup") @@ -1739,7 +1736,6 @@ def search_main( logger.debug("Running clustering on " + str(len(fas4clus)) + " fastas") logger.info("CRAP") - ome2genes = {} fas4trees = {k: v for k, v in sorted(fas4trees.items(), key=lambda x: len(x[1]))} for query, query_fa in fas4trees.items(): out_keys = None @@ -1840,7 +1836,6 @@ def search_main( verbose=False, ) query_hits = all_keys - out_query = query + ".outgroup" out_keys = list(set(all_keys).difference(set(in_keys))) logger.debug("" + str(len(in_keys)) + " gene ingroup") if out_keys: @@ -2079,7 +2074,6 @@ def cli(): ) db = mtdb(args.mtdb) - gene0 = input_genes[0] ome = input_genes[0][: input_genes[0].find("_")] if not ome in set(db["ome"]): if Path(args.query).is_file(): diff --git a/mycotools/db2files.py b/mycotools/db2files.py index a4ed619..b393d9e 100755 --- a/mycotools/db2files.py +++ b/mycotools/db2files.py @@ -2,7 +2,6 @@ import logging import os -import re import sys import argparse from datetime import datetime @@ -126,16 +125,6 @@ def cli(): db_path = format_path(args.mtdb) args.output = format_path(args.output, force_dir=True) output_path = prep_output(args.output, cd=False) - args_dict = { - "DATABASE": db_path, - "OUTPUT": output_path, - "ASSEMBLY": args.assembly, - "PROTEOME": args.proteome, - "GFF3": args.gff, - "Print links": args.print, - "Hard copy": args.hard, - "New MTDB": args.new_mtdb, - } filetypes = [] if args.proteome: diff --git a/mycotools/db2hgs.py b/mycotools/db2hgs.py index aa1e7ed..218a11d 100755 --- a/mycotools/db2hgs.py +++ b/mycotools/db2hgs.py @@ -1,6 +1,5 @@ #! /usr/bin/env python3 -import os import sys import shutil import logging @@ -9,10 +8,9 @@ import multiprocessing as mp from statistics import stdev, StatisticsError from collections import defaultdict, Counter -from mycotools.acc2fa import dbmain as acc2fa from mycotools.db2files import soft_main as symlink_files from mycotools.lib.dbtools import mtdb, primaryDB -from mycotools.lib.biotools import fa2dict, dict2fa, fa2dict_accs +from mycotools.lib.biotools import dict2fa, fa2dict_accs from mycotools.lib.kontools import format_path, mkOutput, findExecs, setup_logging from pathlib import Path @@ -444,7 +442,6 @@ def main( hg_dir = nscg_dir else: srch_hgs = schgs - hg_dir = scg_dir for hg in srch_hgs: if not Path(f"{msa_dir}{hg}.mafft.faa").is_file(): mafft_code = align_hg( diff --git a/mycotools/db2microsyntree.py b/mycotools/db2microsyntree.py index 2da3c9b..a0c7038 100755 --- a/mycotools/db2microsyntree.py +++ b/mycotools/db2microsyntree.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import logging -import os import sys import shutil import argparse @@ -10,7 +9,7 @@ import multiprocessing as mp from tqdm import tqdm from itertools import combinations -from collections import defaultdict, Counter +from collections import defaultdict from mycotools.db2files import soft_main as symlink_files from mycotools.db2hgs import id_near_schgs from mycotools.lib.kontools import ( @@ -527,7 +526,6 @@ def main( # cooccur_array = np.load(cc_arr_path + '.npy') ome2pairs = {ome2i[ome]: v for ome, v in ome2pairs.items() if ome in ome2i} - microsynt_dict = {} logger.info("II. Microsynteny tree") if not Path(tree_path).is_file(): nschgs = [] diff --git a/mycotools/db2search.py b/mycotools/db2search.py index 145825f..ded5d0b 100755 --- a/mycotools/db2search.py +++ b/mycotools/db2search.py @@ -12,18 +12,13 @@ # NEED nhmmer option # NEED to .tmp and move files -import os import re import sys -import copy import logging -import datetime import argparse import subprocess import multiprocessing as mp -from io import StringIO from collections import defaultdict -from mycotools.db2files import soft_main as db2files from mycotools.lib.kontools import ( intro, outro, @@ -44,7 +39,7 @@ # from mycotools.extractHmmsearch import main as exHmm from mycotools.acc2fa import dbmain as acc2fa_db, famain as acc2fa_fa from mycotools.utils.extractHmmsearch import main as exHmm -from mycotools.utils.extractHmmAcc import grabAccs, main as absHmm +from mycotools.utils.extractHmmAcc import grabAccs from pathlib import Path logger = logging.getLogger(__name__) @@ -199,7 +194,6 @@ def compile_hmmalign_cmds(output, accessions): hmm = output + "/hmms/" + acc + ".hmm" align = output + "/aligns/" + acc + ".stockholm" conv = output + "/aligns/" + acc + ".phylip" - trim = output + "/trimmed/" + acc + ".clipkit.fa" if Path(conv).is_file(): with open(conv, "r") as raw: data = raw.read() @@ -641,7 +635,6 @@ def compileResults(res_dict, skip=[]): for hit in res_dict[i]: query = hit[0] subject = hit[1] - pident = hit[2] start = hit[-4] end = hit[-3] if query not in output_res: @@ -1134,7 +1127,6 @@ def parseDBout(db, file_, bitscore=0, pident=0, ppos=0, max_hits=None): ome_results[ome].append(data) x_omes = set(db["ome"]) - omes_results = {x: ome_results[x] for x in ome_results if x in x_omes} if max_hits: out_results = {} @@ -1244,7 +1236,6 @@ def blast_main( seq_type = "prot" biotype = "faa" elif blast in {"blastx", "blastn"}: - seq_type = "nucl" biotype = "fna" else: logger.error("invalid search binary: " + blast) diff --git a/mycotools/fa2clus.py b/mycotools/fa2clus.py index 5de24a9..e097c00 100755 --- a/mycotools/fa2clus.py +++ b/mycotools/fa2clus.py @@ -10,11 +10,7 @@ import sys import copy import shutil -import string -import random -import tempfile import argparse -import itertools import subprocess import pandas as pd from collections import defaultdict diff --git a/mycotools/fa2mass.py b/mycotools/fa2mass.py index b39fd57..4c630a2 100755 --- a/mycotools/fa2mass.py +++ b/mycotools/fa2mass.py @@ -1,6 +1,5 @@ #! /usr/bin/env python3 -import re import sys import logging from mycotools.lib.kontools import sys_start, format_path, fmt_float, setup_logging diff --git a/mycotools/fa2tree.py b/mycotools/fa2tree.py index 0c51538..18be9ef 100755 --- a/mycotools/fa2tree.py +++ b/mycotools/fa2tree.py @@ -12,7 +12,6 @@ import argparse import subprocess import contextlib -import multiprocessing as mp from collections import defaultdict from mycotools.lib.kontools import ( collect_files, @@ -143,7 +142,6 @@ def run_clipkit( ] else: cmd = ["clipkit", mafft_name, "--output", clipkit_out_name] - mode = "smart-gap" gappy = None # execute immediately @@ -226,7 +224,6 @@ def run_mf( """Run ModelFinder on its own in preparation for multigene phylogeny reconstruction""" # set the CPUs for each ModelFinder run arbitrarily - cpus_per_cmd = 3 # determine the concurrent ModelFinder runs that are possible concurrent_cmds = round((cpus - 1) / 4 - 0.5) # round down if not concurrent_cmds: diff --git a/mycotools/gff2seq.py b/mycotools/gff2seq.py index acff629..ad29086 100755 --- a/mycotools/gff2seq.py +++ b/mycotools/gff2seq.py @@ -7,7 +7,7 @@ from Bio.Seq import Seq from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.lib.biotools import fa2dict, gff2list, gff3Comps, dict2fa -from mycotools.lib.kontools import format_path, sys_start, stdin2str, setup_logging +from mycotools.lib.kontools import format_path, stdin2str, setup_logging logger = logging.getLogger(__name__) @@ -142,7 +142,6 @@ def grabCoords(cdss): pos_dict[seqid][gene] = [] pos_dict[seqid][gene].append([int(entry["start"]) - 1, int(entry["end"])]) elif entry["strand"].rstrip() == "-": - negseq = seqid if seqid not in neg_dict: neg_dict[seqid], neg_dict[seqid][gene] = {}, [] elif gene not in neg_dict[seqid]: diff --git a/mycotools/jgiDwnld.py b/mycotools/jgiDwnld.py index aaea565..eda9ace 100755 --- a/mycotools/jgiDwnld.py +++ b/mycotools/jgiDwnld.py @@ -12,7 +12,6 @@ import sys import time import logging -import getpass import argparse import subprocess import pandas as pd diff --git a/mycotools/lib/biotools.py b/mycotools/lib/biotools.py index cae4e0f..d7d7365 100755 --- a/mycotools/lib/biotools.py +++ b/mycotools/lib/biotools.py @@ -3,7 +3,6 @@ # NEED to convert gff list to appropriate types import re -import sys from collections import defaultdict from itertools import chain diff --git a/mycotools/lib/dbtools.py b/mycotools/lib/dbtools.py index e3b43f6..e9809a6 100755 --- a/mycotools/lib/dbtools.py +++ b/mycotools/lib/dbtools.py @@ -545,7 +545,6 @@ def encrypt_pw( from cryptography.fernet import Fernet from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - from cryptography.hazmat.backends import default_backend kdf = PBKDF2HMAC( algorithm=hashes.SHA256(), @@ -575,7 +574,6 @@ def loginCheck(info_path="~/.mycotools/mtdb_key", ncbi=True, jgi=True, encrypt=F if Path(format_path(info_path)).is_file(): from cryptography.fernet import Fernet from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes kdf = PBKDF2HMAC( @@ -686,7 +684,7 @@ def primaryDB(path="$MYCODB", verbose=True): # returns database dataframe def db2df(data, stdin=False): """Deprecated legacy Pandas implementation of MTDB import""" - import pandas as pd, pandas + import pandas as pd columns = mtdb.columns if isinstance(data, mtdb): @@ -737,7 +735,7 @@ def df2std(df): # if rescue is set to 0, do not output database if output dir does not exit def df2db(df, db_path, header=False, overwrite=False, std_col=True, rescue=True): """Deprecated output pandas MTDB implementation to file""" - import pandas as pd, pandas + import pandas as pd df = df.set_index("ome") df = df.sort_index() diff --git a/mycotools/manage_mtdb.py b/mycotools/manage_mtdb.py index 1623178..626ba1f 100755 --- a/mycotools/manage_mtdb.py +++ b/mycotools/manage_mtdb.py @@ -5,7 +5,7 @@ import logging import argparse from mycotools.lib.dbtools import loginCheck, primaryDB, mtdb, encrypt_pw -from mycotools.lib.kontools import format_path, read_json, collect_files, setup_logging +from mycotools.lib.kontools import format_path, read_json, setup_logging from pathlib import Path logger = logging.getLogger(__name__) diff --git a/mycotools/ncbiDwnld.py b/mycotools/ncbiDwnld.py index 9bebbeb..96388dd 100755 --- a/mycotools/ncbiDwnld.py +++ b/mycotools/ncbiDwnld.py @@ -7,7 +7,6 @@ import os import re import sys -import gzip import json import time import shutil @@ -16,9 +15,7 @@ import zipfile import argparse import subprocess -import numpy as np import pandas as pd -from contextlib import closing from tqdm import tqdm from Bio import Entrez from datetime import datetime @@ -33,7 +30,7 @@ split_input, setup_logging, ) -from mycotools.lib.dbtools import log_editor, loginCheck, mtdb, read_tax +from mycotools.lib.dbtools import log_editor, loginCheck, mtdb from pathlib import Path logger = logging.getLogger(__name__) @@ -42,9 +39,8 @@ def ncbidb2df(data, stdin=False): - import pandas as pd, pandas + import pandas as pd - columns = mtdb.columns if isinstance(data, mtdb): db_df = pd.DataFrame(data.reset_index()) elif not stdin: @@ -244,7 +240,6 @@ def collect_assembly_accs( record_info = record["DocumentSummarySet"]["DocumentSummary"][0] assemblyID = record_info["AssemblyAccession"] - esc_count = 0 log_editor( output_path + "ncbiDwnld.log", str(new_acc), diff --git a/mycotools/ncbi_dwnld_fallback.py b/mycotools/ncbi_dwnld_fallback.py index 1ed2fe0..7d9350b 100644 --- a/mycotools/ncbi_dwnld_fallback.py +++ b/mycotools/ncbi_dwnld_fallback.py @@ -8,7 +8,6 @@ import os import re import sys -import gzip import time import shutil import urllib @@ -16,9 +15,7 @@ import requests import argparse import subprocess -import numpy as np import pandas as pd -from contextlib import closing from tqdm import tqdm from Bio import Entrez from datetime import datetime @@ -30,16 +27,15 @@ findExecs, setup_logging, ) -from mycotools.lib.dbtools import log_editor, loginCheck, mtdb, read_tax +from mycotools.lib.dbtools import log_editor, loginCheck, mtdb from pathlib import Path logger = logging.getLogger(__name__) def ncbidb2df(data, stdin=False): - import pandas as pd, pandas + import pandas as pd - columns = mtdb.columns if isinstance(data, mtdb): db_df = pd.DataFrame(data.reset_index()) elif not stdin: @@ -279,7 +275,6 @@ def collect_ftps( strain = "" # populate a fallback strain record_info = record["DocumentSummarySet"]["DocumentSummary"][0] - assemblyID = record_info["AssemblyAccession"] org = record_info["SpeciesName"].split() genus = org[0] if len(org) > 2: @@ -317,7 +312,6 @@ def collect_ftps( failed.append([accession, str(row["version"])]) continue - esc_count = 0 ass_md5, gff_md5, trans_md5, prot_md5, md5s = "", "", "", "", {} basename = Path(ftp_path).name diff --git a/mycotools/predb2mtdb.py b/mycotools/predb2mtdb.py index 482ec75..b9b1ac4 100755 --- a/mycotools/predb2mtdb.py +++ b/mycotools/predb2mtdb.py @@ -21,7 +21,7 @@ gff2Comps, gtfComps, ) -from mycotools.lib.dbtools import mtdb, primaryDB, loginCheck +from mycotools.lib.dbtools import mtdb, primaryDB from mycotools.utils.gtf2gff3 import main as gtf2gff3 from mycotools.utils.curGFF3 import main as curGFF3 from mycotools.utils.gff2gff3 import main as gff2gff3 @@ -449,7 +449,6 @@ def cur_fna(cur_raw_fna_path, uncur_raw_fna_path, ome): ver_num = ome_ver[2] else: less_ome = ome - ver_num = 0 with open(cur_raw_fna_path + ".tmp", "w") as out: with open(uncur_raw_fna_path, "r") as in_: for line in in_: @@ -627,7 +626,6 @@ def gff_mngr(ome, gff, cur_path, source, assembly_accession): ver_search = re.search(r"(.{6}\d+)\.(\d+)", ome) if ver_search is not None: less_ome = ver_search[1] - ome_ver = ver_search[2] else: less_ome = ome for line in gff: diff --git a/mycotools/treetools.py b/mycotools/treetools.py index c6976d6..2c07d6f 100755 --- a/mycotools/treetools.py +++ b/mycotools/treetools.py @@ -5,7 +5,7 @@ import sys import argparse from itertools import chain -from cogent3 import PhyloNode, load_tree +from cogent3 import load_tree from mycotools.lib.kontools import format_path, split_input, setup_logging from mycotools.lib.dbtools import mtdb from pathlib import Path diff --git a/mycotools/update_mtdb.py b/mycotools/update_mtdb.py index ecb2661..4934fea 100755 --- a/mycotools/update_mtdb.py +++ b/mycotools/update_mtdb.py @@ -16,12 +16,8 @@ import os import re import sys -import time import json -import base64 import shutil -import getpass -import hashlib import zipfile import requests import argparse @@ -69,8 +65,6 @@ from mycotools.utils.jgi2db import main as jgi2db from mycotools.predb2mtdb import main as predb2mtdb from mycotools.predb2mtdb import predb_headers, read_predb, gen_omes -from mycotools.assemblyStats import main as assStats -from mycotools.annotationStats import main as annStats from pathlib import Path logger = logging.getLogger(__name__) @@ -852,7 +846,6 @@ def ref_update( # run JGI if jgi and len(jgi_df) > 0: logger.info("Assimilating MycoCosm") - jgi_db_path = update_path + date + ".jgi.mtdb" jgi_predb_path = update_path + date + ".jgi.predb2.mtdb" if not Path(jgi_predb_path).is_file(): @@ -1012,7 +1005,6 @@ def ref_update( tax_dicts=tax_dicts, ) new_mtdb, genus_dicts = assimilate_tax(new_mtdb, tax_dicts) - dupFiles = {"fna": {}, "faa": {}, "gff3": {}} for ome, row in update_mtdb.items(): if row["genus"] in genus_dicts: @@ -1293,7 +1285,6 @@ def rogue_update( else: jgi_mtdb = None new_db = db - new_dups = duplicates logger.info("Assimilating NCBI (10 download/second w/API key, 3 w/o)") new_db["version"] = new_db["version"].astype(str) @@ -1386,7 +1377,6 @@ def rogue_update( output_path=tax_path, ) new_mtdb, genus_dicts = assimilate_tax(new_mtdb, tax_dicts) - dupFiles = {"fna": {}, "faa": {}, "gff3": {}} if jgi_mtdb and ncbi_mtdb: update_mtdb = mtdb( diff --git a/mycotools/utils/curGFF3.py b/mycotools/utils/curGFF3.py index 7adbdba..31a20eb 100755 --- a/mycotools/utils/curGFF3.py +++ b/mycotools/utils/curGFF3.py @@ -349,7 +349,6 @@ def add_missing(gff_list, intron, comps, ome): continue if introns: - exons = {} for gene, intronList in introns.items(): if out_genes[gene]["exon"]: continue diff --git a/mycotools/utils/gff2gff3.py b/mycotools/utils/gff2gff3.py index 910a275..95f412a 100755 --- a/mycotools/utils/gff2gff3.py +++ b/mycotools/utils/gff2gff3.py @@ -5,12 +5,11 @@ import logging import re import sys -import copy import argparse from collections import defaultdict from mycotools.lib.biotools import gff2list, list2gff, gff2Comps, gff3Comps from mycotools.lib.kontools import format_path, setup_logging -from mycotools.utils.gtf2gff3 import add_genes, remove_start_stop +from mycotools.utils.gtf2gff3 import add_genes from mycotools.utils.curGFF3 import rename_and_organize logger = logging.getLogger(__name__) diff --git a/mycotools/utils/gtf2gff3.py b/mycotools/utils/gtf2gff3.py index c7d8d2b..e3c8cb0 100755 --- a/mycotools/utils/gtf2gff3.py +++ b/mycotools/utils/gtf2gff3.py @@ -590,7 +590,6 @@ def curate(gff, prefix=None, failed=set(), comps=gtfComps()): entry["attributes"] += f";Alias={alias_dict[trans]}" exon_check[trans] += 1 elif entry["type"] == "CDS": - alias = alias_dict[trans] trans = tran_comp.search(entry["attributes"])[1] cds = cds_check[trans] + 1 entry["attributes"] = f"ID={trans}.cds{cds};Parent=" diff --git a/mycotools/utils/jgi2db.py b/mycotools/utils/jgi2db.py index 9390c59..c2e37ad 100755 --- a/mycotools/utils/jgi2db.py +++ b/mycotools/utils/jgi2db.py @@ -2,18 +2,13 @@ import logging import os -import re import sys import copy import time -import getpass import datetime import argparse -import subprocess import pandas as pd import numpy as np -from io import StringIO -from mycotools.predb2mtdb import main as predb2mtdb from mycotools.lib.kontools import intro, outro, setup_logging from mycotools.lib.dbtools import db2df, df2db, readLog, log_editor from mycotools.jgiDwnld import jgi_login as jgi_login @@ -259,7 +254,6 @@ def main( ome_col = "assembly_acc" elif "portal" in jgi_df.columns: ome_col = "portal" - name_col = "name" else: logger.debug(spacer + "invalid MycoCosm tsv headers") sys.exit(3) @@ -513,9 +507,6 @@ def cli(): with open(args.login, "r") as raw: prep = raw.read() data = [x.split("\t") for x in prep.split("\n")] - user = data[0][0] - pwd = data[0][1] - email = data[1][0] apikey = None if len(data[1]) > 1: if data[1][1] != "": diff --git a/mycotools/utils/ncbi2db.py b/mycotools/utils/ncbi2db.py index 5c84ac4..5d4e5a3 100755 --- a/mycotools/utils/ncbi2db.py +++ b/mycotools/utils/ncbi2db.py @@ -10,17 +10,9 @@ import logging import os import re -import sys import copy -import argparse -import subprocess -import numpy as np -import pandas as pd from datetime import datetime -from mycotools.lib.kontools import intro, outro -from mycotools.lib.dbtools import db2df, df2db, primaryDB from mycotools.ncbiDwnld import main as ncbi_dwnld -from mycotools.predb2mtdb import main as predb2mtdb logger = logging.getLogger(__name__) diff --git a/test/unit/README.md b/test/unit/README.md new file mode 100644 index 0000000..d8e14b4 --- /dev/null +++ b/test/unit/README.md @@ -0,0 +1,51 @@ +# mycotools unit-test scaffold + +Offline, quick unit tests for the mycotools scripts. No JGI/NCBI login, no +downloads, no genome assimilation — every test exercises argument handling, +argparse surfaces, or login-free helper functions. + +## Run + +```bash +micromamba run -n mycotools pytest test/unit/ -q # this scaffold +micromamba run -n mycotools pytest test/ -q # scaffold + update_mtdb suite +``` + +(~9 s.) + +## Layout + +| File | What it is | +| --- | --- | +| `conftest.py` | Shared fixtures: `offline_env` (autouse; strips `MYCODB`, sets dummy `MYCO*` prefixes), path fixtures (`ust_mtdb`, `reference_mtdb`, `repo_root`, `test_data_dir`), and a `run_cli` subprocess helper. | +| `test_cli_smoke.py` | Data-driven **import** + **`--help` doesn't crash** checks for every entry-point script. The backbone: one parametrized case per script. | +| `test_lib_biotools.py` | A concrete, worked example of deeper unit tests (pure `fa2dict`/`dict2fa` parsers) — the pattern to copy for real logic. | +| `_template.py` | Copy to `test_