<commit_before>import argparse
import csv
import math
import os
import sys

from rdkit import Chem

import six

import tqdm

from ._base import Calculator, all_descriptors, get_descriptors_from_module


def smiles_parser(ifile):
    with ifile as f:
        for line in f:
            line = line.strip().split()
            if len(line) == 1:
                smi = line[0]
                name = smi
            else:
                smi = line[0]
                name = ' '.join(line[1:])

            mol = Chem.MolFromSmiles(smi)

            if mol is None:
                sys.stderr.write('read failure: {}\n'.format(name))
                sys.stderr.flush()
                continue

            mol.SetProp('_Name', name)
            yield mol


def sdf_parser(path):
    for mol in Chem.SDMolSupplier(path, removeHs=False):
        yield mol


def mol_parser(path):
    mol = Chem.MolFromMolFile(path, removeHs=False)
    if mol is None:
        return ()

    if mol.GetProp('_Name') == '':
        mol.SetProp(
            '_Name',
            os.path.splitext(os.path.basename(path))[0],
        )

    return [mol]


def dir_parser(d, fmt):
    for root, dirs, files in os.walk(d):
        for f in files:
            for m in file_parser(os.path.join(root, f), fmt):
                yield m


def file_parser(ifile, fmt):
    if fmt == 'smi':
        if isinstance(ifile, six.string_types):
            ifile = open(ifile)

        it = smiles_parser(ifile)
    elif fmt in 'sdf':
        it = sdf_parser(ifile)
    elif fmt == 'mol':
        it = mol_parser(ifile)
    elif fmt == 'auto':
        if ifile == sys.stdin:
            it = file_parser(ifile, 'smi')
        elif os.path.isdir(ifile):
            it = dir_parser(ifile, fmt)
        else:
            ext = os.path.splitext(ifile)[1][1:]
            it = file_parser(ifile, ext)
    else:
        raise ValueError('unknown format: {!r}'.format(fmt))

    return (mol for mol in it if mol is not None)


class DummyBar(object):
    def __init__(self, *args, **kws):
        pass

    def __enter__(self):
        return self

    def __exit__(self, *args, **kws):
        pass

    def update(self, *args, **kws):
        pass


def main(descs, prog=None):
    parser_options = dict()
    if prog is not None:
        parser_options['prog'] = '{} -m {}'.format(os.path.basename(sys.executable), prog)

    parser = argparse.ArgumentParser(**parser_options)

    parser.add_argument(
        '-i', '--input', metavar='PATH',
        type=str, default=sys.stdin,
        help='input file or directory(default: stdin)',
    )

    parser.add_argument(
        '-f', '--from', metavar='TYPE',
        default='auto', choices=['auto', 'smi', 'sdf', 'mol'],
        help='input filetype(one of %(choices)s, default: %(default)s)',
    )

    parser.add_argument(
        '-o', '--output', metavar='PATH',
        type=str, default=None,
        help='output csv file(default: stdout)'
    )

    parser.add_argument(
        '-p', '--processes', metavar='N',
        default=None, type=int,
        help='number of processes to use(default: number of threads)',
    )

    parser.add_argument(
        '-q', '--quiet',
        default=False, action='store_true',
        help="hide progress bar",
    )

    parser.add_argument(
        '-s', '--stream',
        default=False, action='store_true',
        help='stream read',
    )

    parser.add_argument(
        '-3', '--with-3D',
        default=False, action='store_true',
        help='calculate 3D descriptor(require sdf or mol file)'
    )

    args = parser.parse_args()

    if args.input == sys.stdin and args.input.isatty():
        sys.exit(parser.print_help())

    if args.output is None:
        args.output = sys.stdout
    elif six.PY3:
        args.output = open(args.output, 'w', newline='')
    else:
        args.output = open(args.output, 'wb')

    if args.output.isatty():
        args.quiet = True

    mols = file_parser(args.input, getattr(args, 'from'))

    if not args.stream:
        mols = list(mols)
        N = len(mols)<commit_msg>__main__: make input positional argument<commit_after>import argparse
import csv
import math
import os
import sys

from rdkit import Chem

import six

import tqdm

from ._base import Calculator, all_descriptors, get_descriptors_from_module


def smiles_parser(ifile):
    with ifile as f:
        for line in f:
            line = line.strip().split()
            if len(line) == 1:
                smi = line[0]
                name = smi
            else:
                smi = line[0]
                name = ' '.join(line[1:])

            mol = Chem.MolFromSmiles(smi)

            if mol is None:
                sys.stderr.write('read failure: {}\n'.format(name))
                sys.stderr.flush()
                continue

            mol.SetProp('_Name', name)
            yield mol


def sdf_parser(path):
    for mol in Chem.SDMolSupplier(path, removeHs=False):
        yield mol


def mol_parser(path):
    mol = Chem.MolFromMolFile(path, removeHs=False)
    if mol is None:
        return ()

    if mol.GetProp('_Name') == '':
        mol.SetProp(
            '_Name',
            os.path.splitext(os.path.basename(path))[0],
        )

    return [mol]


def dir_parser(d, fmt):
    for root, dirs, files in os.walk(d):
        for f in files:
            for m in file_parser(os.path.join(root, f), fmt):
                yield m


def file_parser(ifile, fmt):
    if fmt == 'smi':
        if isinstance(ifile, six.string_types):
            ifile = open(ifile)

        it = smiles_parser(ifile)
    elif fmt in 'sdf':
        it = sdf_parser(ifile)
    elif fmt == 'mol':
        it = mol_parser(ifile)
    elif fmt == 'auto':
        if ifile == sys.stdin:
            it = file_parser(ifile, 'smi')
        elif os.path.isdir(ifile):
            it = dir_parser(ifile, fmt)
        else:
            ext = os.path.splitext(ifile)[1][1:]
            it = file_parser(ifile, ext)
    else:
        raise ValueError('unknown format: {!r}'.format(fmt))

    return (mol for mol in it if mol is not None)


class DummyBar(object):
    def __init__(self, *args, **kws):
        pass

    def __enter__(self):
        return self

    def __exit__(self, *args, **kws):
        pass

    def update(self, *args, **kws):
        pass


def main(descs, prog=None):
    parser_options = dict()
    if prog is not None:
        parser_options['prog'] = '{} -m {}'.format(os.path.basename(sys.executable), prog)

    parser = argparse.ArgumentParser(**parser_options)

    parser.add_argument(
        'INPUT', type=str, default=sys.stdin,
        help='input file or directory(default: stdin)',
    )

    parser.add_argument(
        '-f', '--from', metavar='TYPE',
        default='auto', choices=['auto', 'smi', 'sdf', 'mol'],
        help='input filetype(one of %(choices)s, default: %(default)s)',
    )

    parser.add_argument(
        '-o', '--output', metavar='PATH',
        type=str, default=None,
        help='output csv file(default: stdout)'
    )

    parser.add_argument(
        '-p', '--processes', metavar='N',
        default=None, type=int,
        help='number of processes to use(default: number of threads)',
    )

    parser.add_argument(
        '-q', '--quiet',
        default=False, action='store_true',
        help="hide progress bar",
    )

    parser.add_argument(
        '-s', '--stream',
        default=False, action='store_true',
        help='stream read',
    )

    parser.add_argument(
        '-3', '--with-3D',
        default=False, action='store_true',
        help='calculate 3D descriptor(require sdf or mol file)'
    )

    args = parser.parse_args()

    if args.INPUT == sys.stdin and args.INPUT.isatty():
        sys.exit(parser.print_help())

    if args.output is None:
        args.output = sys.stdout
    elif six.PY3:
        args.output = open(args.output, 'w', newline='')
    else:
        args.output = open(args.output, 'wb')

    if args.output.isatty():
        args.quiet = True

    mols = file_parser(args.INPUT, getattr(args, 'from'))

    if not args.stream:
        mols = list(mols)
        N = len(mols)
{'fraction_non_alphanumeric': 0.13611348905466553, 'fraction_numerical': 0.002690473278708573, 'mean_word_length': 2.0446760982874164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1159532', 'n_tokens_mistral': 2741, 'n_tokens_neox': 2626, 'n_words': 647}