<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
''' Parallel NCBI download script '''
import sys, os, argparse
from subprocess import Popen, PIPE
from pipes import quote

from isolates import __version__, ceil
from source import acctypes
from isolates.metadata import ExtractExperimentIDs_acc, ExtractExperimentIDs_tax

def parse_args(args):
    """
    Parse command line parameters

    :param args: command line parameters as list of strings
    :return: command line parameters as :obj:`argparse.Namespace`
    """
    parser = argparse.ArgumentParser(
        description="Download script of isolates from" +
                    "ENA taxonomy or Accession list")
    parser.add_argument(
        '-v',
        '--version',
        action='version',
        version='isolates {ver}'.format(ver=__version__))
    parser.add_argument(
        '-a',
        metavar=('PATH'),
        help='Format: [PATH]\n' +
             'to file containing list of ACCESSION IDs, 1 per line\n' +
             'Name of the file is used to identify the isolates downloaded.'
    )
    parser.add_argument(
        '-t',
        nargs=1,
        metavar=('TAXID'),
        help='Tax ID from ENA data archive'
    )
    parser.add_argument(
        '-m',
        metavar=('PATH'),
        default=None,
        help='JSON file with seed attributes and mandatory fields\n'
    )
    parser.add_argument(
        '-p',
        '--preserve',
        action="store_true",
        dest="preserve",
        default=False,
        help='preserve any existing SRA and fastq files\n'
    )
    parser.add_argument(
        '--all_runs_as_samples',
        action="store_true",
        dest="all_runs_as_samples",
        default=False,
        help=('Treat all runs associated to a sample as separate samples. '
              'Default is to combine them into one run.\n')
    )
    parser.add_argument(
        '-n',
        '--nodes',
        dest="nodes",
        default=1,
        help=('Number of parallel batch jobs requested [default: 1]\n')
    )
    parser.add_argument(
        '-out',
        metavar=('OUTPUT'),
        required=True,
        help='Path to save isolates'
    )
    return parser.parse_args(args)

def SetupParallelDownload(accession_list):
    """ Expand list of accession IDs to experiment or lower, and devide into
    parallel batch jobs
    """
    experiments = []
    failed_accession = []
    with open(accession_list, 'r') as f:
        for l in f:
            accession = l.strip()
            if accession == '': continue
            # Determine accession type<commit_msg>clean up and debug continued...<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
''' Parallel NCBI download script '''
import sys, os, argparse
from subprocess import Popen, PIPE
from pipes import quote

from isolates import __version__, ceil
from source import acctypes
from isolates.metadata import ExtractExperimentIDs_acc, ExtractExperimentIDs_tax

def parse_args(args):
    ''' Parse command line parameters '''
    parser = argparse.ArgumentParser(
        description=('Download samples from NCBI through either a taxonomy or '
                     'accession ID input'))
    parser.add_argument(
        '-v',
        '--version',
        action='version',
        version='NCBI Downloader {ver}'.format(ver=__version__))
    parser.add_argument(
        '-a',
        metavar=('ACCESSION'),
        help=('Input should be a path to file containing accession IDs or a '
              'comma-separated string of accession IDs.\n'
              'The file may only contain one accession ID per line.\n')
    )
    parser.add_argument(
        '-t',
        metavar=('TAXID'),
        help=('Input should be a path to file containing taxonomy IDs or a '
              'comma-separated string of taxonomy IDs.\n'
              'The file may only contain one taxonomy ID per line.\n')
    )
    parser.add_argument(
        '-m',
        default=None,
        help=('JSON file with seed attributes (default fields and values) and '
              'mandatory fields.\n')
    )
    parser.add_argument(
        '-p',
        '--preserve',
        action="store_true",
        dest="preserve",
        default=False,
        help='Preserve any existing fastq files.\n'
    )
    parser.add_argument(
        '--all_runs_as_samples',
        action="store_true",
        dest="all_runs_as_samples",
        default=False,
        help=('Treat all runs associated to a sample as separate samples. '
              'Default is to combine them into one run.\n')
    )
    parser.add_argument(
        '-n',
        '--nodes',
        dest="nodes",
        default=1,
        help='Number of parallel batch jobs requested [default: 1]\n'
    )
    parser.add_argument(
        '-out',
        metavar=('OUTPUT'),
        required=True,
        help='output directory name.'
    )
    return parser.parse_args(args)

def SetupParallelDownload(accession_list):
    ''' Expand list of accession IDs to experiment or lower, and devide into
    parallel batch jobs
    '''
    experiments = []
    failed_accession = []
    with open(accession_list, 'r') as f:
        for l in f:
            accession = l.strip()
            if accession == '': continue
            # Determine accession type
{'fraction_non_alphanumeric': 0.10758177349215353, 'fraction_numerical': 0.0015125732652675364, 'mean_word_length': 2.225609756097561, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7194659', 'n_tokens_mistral': 1507, 'n_tokens_neox': 1455, 'n_words': 449}