<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import sys
import argparse
import subprocess
import time
import logging


# Create logger
logger = logging.getLogger(__name__)

# Get program filename
program_filename = os.path.basename(sys.argv[0])

# Get MATAM root dir absolute path
matam_db_prepro_bin = os.path.realpath(sys.argv[0])
matam_bin_dir = os.path.dirname(matam_db_prepro_bin)
matam_root_dir = os.path.dirname(matam_bin_dir)

# Get all dependencies bin
matam_script_dir = os.path.join(matam_root_dir, 'scripts')
extract_taxo_bin = os.path.join(matam_script_dir, 'extract_taxo_from_fasta.py')
replace_Ns_bin = os.path.join(matam_script_dir, 'replace_Ns_by_rand_nu.py')
sort_fasta_bin = os.path.join(matam_script_dir, 'sort_fasta_by_length.py')
fasta_name_filter_bin = os.path.join(matam_script_dir, 'fasta_name_filter.py')
clean_name_bin = os.path.join(matam_script_dir, 'fasta_clean_name.py')

sortmerna_bin_dir = os.path.join(matam_root_dir, 'sortmerna')
indexdb_bin = os.path.join(sortmerna_bin_dir, 'indexdb_rna')

sumaclust_bin_dir = os.path.join(matam_root_dir, 'sumaclust')
sumaclust_bin = os.path.join(sumaclust_bin_dir, 'sumaclust')

# Define a null file handle
FNULL = open(os.devnull, 'w')


class DefaultHelpParser(argparse.ArgumentParser):
    """
    This is a slightly modified argparse parser to display the full help
    on parser error instead of only usage
    """
    def error(self, message):
        sys.stderr.write('\nError: %s\n\n' % message)
        self.print_help()
        sys.exit(2)


def parse_arguments():
    """
    Parse the command line, and check if arguments are correct
    """
    # Initiate argument parser
    parser = DefaultHelpParser(description='MATAM db preprocessing',
                               # to precisely format help display
                               formatter_class=lambda prog: argparse.HelpFormatter(prog, width=120, max_help_position=80))

    # Main parameters
    group_main = parser.add_argument_group('Main parameters')
    # -i / --input_ref
    group_main.add_argument('-i', '--input_ref',
                            action = 'store',
                            metavar = 'INREF',
                            type = str,
                            required = True,
                            help = 'Input reference file (fasta format). '
                                   'Silva-formated taxonomies will be used if available')
    # -d / --db_dir
    group_main.add_argument('-d', '--db_dir',
                            action = 'store',
                            metavar = 'DBDIR',
                            type = str,
                            default = os.getcwd(),
                            help = 'Database output directory. '
                                   'Default is cwd')
    # -v / --verbose
    group_main.add_argument('-v', '--verbose',
                            action = 'store_true',
                            help = 'Increase verbosity')

    # Performance parameters
    group_perf = parser.add_argument_group('Performance')
    # --cpu
    group_perf.add_argument('--cpu',
                            action = 'store',
                            metavar = 'CPU',
                            type = int,
                            default = 1,
                            help = 'Max number of CPU to use. '
                                   'Default is %(default)s cpu')
    # --max_memory
    group_perf.add_argument('--max_memory',
                            action = 'store',
                            metavar = 'MAXMEM',
                            type = int,
                            default = 4000,
                            help = 'Maximum memory to use (in MBi). '
                                   'Default is %(default)s MBi')

    # Advanced parameters
    group_adv = parser.add_argument_group('Advanced parameters')
    # --remove_Ns
    group_adv.add_argument('--remove_Ns',
                           action = 'store_true',
                           help = 'Remove sequences with Ns. '
                                  'Default is replacing Ns with random nucleotides')
    # --clustering_id_threshold
    group_adv.add_argument('--clustering_id_threshold',
                           action = 'store',
                           metavar = 'REAL',
                           type = float,
                           default = 0.95,
                           help = 'Identity threshold for clustering. '
                                  'Default is %(default)s')
    # --by_kingdoms
    group_adv.add_argument('--by_kingdom',
                           action = 'store_true',
                           help = 'Perform clustering by kingdom')
    # --kingdoms
    group_adv.add_argument('--kingdoms',
                           action = 'store',
                           metavar = 'STR',
                           type = str,
                           nargs = '+',
                           default = ['archaea', 'bacteria', 'eukaryota'],
                           help = 'Kingdoms to clusterize the DB for. '
                                  'Default is %(default)s')
    # -o / --out_db_name
    group_adv.add_argument('-o', '--out_db_name',
                           action = 'store',
                           metavar = 'OUTNAME',
                           type = str,
                           help = 'Output MATAM db name. '
                                  'Default is composed from parameters')
    # --keep_tmp
    group_adv.add_argument('--keep_tmp',
                            action = 'store_true',
                            help = 'Do not remove tmp files')
    # --debug
    group_adv.add_argument('--debug',
                            action = 'store_true',
                            help = 'Output debug infos')

    #
    args = parser.parse_args()

    # Arguments checking
    if args.clustering_id_threshold < 0 or args.clustering_id_threshold > 1:
        parser.print_help()
        raise Exception("clustering id threshold not in range [0,1]")

    # Set verbose if debug
    if args.debug:
        args.verbose = True

    # Get absolute path for all arguments
    args.input_ref = os.path.abspath(args.input_ref)
    args.db_dir = os.path.abspath(args.db_dir)
<commit_msg>Remove short option for out db name<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import sys
import argparse
import subprocess
import time
import logging


# Create logger
logger = logging.getLogger(__name__)

# Get program filename
program_filename = os.path.basename(sys.argv[0])

# Get MATAM root dir absolute path
matam_db_prepro_bin = os.path.realpath(sys.argv[0])
matam_bin_dir = os.path.dirname(matam_db_prepro_bin)
matam_root_dir = os.path.dirname(matam_bin_dir)

# Get all dependencies bin
matam_script_dir = os.path.join(matam_root_dir, 'scripts')
extract_taxo_bin = os.path.join(matam_script_dir, 'extract_taxo_from_fasta.py')
replace_Ns_bin = os.path.join(matam_script_dir, 'replace_Ns_by_rand_nu.py')
sort_fasta_bin = os.path.join(matam_script_dir, 'sort_fasta_by_length.py')
fasta_name_filter_bin = os.path.join(matam_script_dir, 'fasta_name_filter.py')
clean_name_bin = os.path.join(matam_script_dir, 'fasta_clean_name.py')

sortmerna_bin_dir = os.path.join(matam_root_dir, 'sortmerna')
indexdb_bin = os.path.join(sortmerna_bin_dir, 'indexdb_rna')

sumaclust_bin_dir = os.path.join(matam_root_dir, 'sumaclust')
sumaclust_bin = os.path.join(sumaclust_bin_dir, 'sumaclust')

# Define a null file handle
FNULL = open(os.devnull, 'w')


class DefaultHelpParser(argparse.ArgumentParser):
    """
    This is a slightly modified argparse parser to display the full help
    on parser error instead of only usage
    """
    def error(self, message):
        sys.stderr.write('\nError: %s\n\n' % message)
        self.print_help()
        sys.exit(2)


def parse_arguments():
    """
    Parse the command line, and check if arguments are correct
    """
    # Initiate argument parser
    parser = DefaultHelpParser(description='MATAM db preprocessing',
                               # to precisely format help display
                               formatter_class=lambda prog: argparse.HelpFormatter(prog, width=120, max_help_position=80))

    # Main parameters
    group_main = parser.add_argument_group('Main parameters')
    # -i / --input_ref
    group_main.add_argument('-i', '--input_ref',
                            action = 'store',
                            metavar = 'INREF',
                            type = str,
                            required = True,
                            help = 'Input reference file (fasta format). '
                                   'Silva-formated taxonomies will be used if available')
    # -d / --db_dir
    group_main.add_argument('-d', '--db_dir',
                            action = 'store',
                            metavar = 'DBDIR',
                            type = str,
                            default = os.getcwd(),
                            help = 'Database output directory. '
                                   'Default is cwd')
    # -v / --verbose
    group_main.add_argument('-v', '--verbose',
                            action = 'store_true',
                            help = 'Increase verbosity')

    # Performance parameters
    group_perf = parser.add_argument_group('Performance')
    # --cpu
    group_perf.add_argument('--cpu',
                            action = 'store',
                            metavar = 'CPU',
                            type = int,
                            default = 1,
                            help = 'Max number of CPU to use. '
                                   'Default is %(default)s cpu')
    # --max_memory
    group_perf.add_argument('--max_memory',
                            action = 'store',
                            metavar = 'MAXMEM',
                            type = int,
                            default = 4000,
                            help = 'Maximum memory to use (in MBi). '
                                   'Default is %(default)s MBi')

    # Advanced parameters
    group_adv = parser.add_argument_group('Advanced parameters')
    # --remove_Ns
    group_adv.add_argument('--remove_Ns',
                           action = 'store_true',
                           help = 'Remove sequences with Ns. '
                                  'Default is replacing Ns with random nucleotides')
    # --clustering_id_threshold
    group_adv.add_argument('--clustering_id_threshold',
                           action = 'store',
                           metavar = 'REAL',
                           type = float,
                           default = 0.95,
                           help = 'Identity threshold for clustering. '
                                  'Default is %(default)s')
    # --by_kingdoms
    group_adv.add_argument('--by_kingdom',
                           action = 'store_true',
                           help = 'Perform clustering by kingdom')
    # --kingdoms
    group_adv.add_argument('--kingdoms',
                           action = 'store',
                           metavar = 'STR',
                           type = str,
                           nargs = '+',
                           default = ['archaea', 'bacteria', 'eukaryota'],
                           help = 'Kingdoms to clusterize the DB for. '
                                  'Default is %(default)s')
    # -o / --out_db_name
    group_adv.add_argument('--out_db_name',
                           action = 'store',
                           metavar = 'OUTNAME',
                           type = str,
                           help = 'Output MATAM db name. '
                                  'Default is composed from parameters')
    # --keep_tmp
    group_adv.add_argument('--keep_tmp',
                            action = 'store_true',
                            help = 'Do not remove tmp files')
    # --debug
    group_adv.add_argument('--debug',
                            action = 'store_true',
                            help = 'Output debug infos')

    #
    args = parser.parse_args()

    # Arguments checking
    if args.clustering_id_threshold < 0 or args.clustering_id_threshold > 1:
        parser.print_help()
        raise Exception("clustering id threshold not in range [0,1]")

    # Set verbose if debug
    if args.debug:
        args.verbose = True

    # Get absolute path for all arguments
    args.input_ref = os.path.abspath(args.input_ref)
    args.db_dir = os.path.abspath(args.db_dir)

{'fraction_non_alphanumeric': 0.09801744647105472, 'fraction_numerical': 0.0034892942109436957, 'mean_word_length': 1.517668197244959, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5196014', 'n_tokens_mistral': 3565, 'n_tokens_neox': 3448, 'n_words': 779}