<commit_before>#!/usr/bin/env python
#
# Copyright 2009 Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A command line parsing module that lets modules define their own options.

Each module defines its own options, e.g.::

    from tornado.options import define, options

    define("mysql_host", default="127.0.0.1:3306", help="Main user DB")
    define("memcache_hosts", default="127.0.0.1:11011", multiple=True,
           help="Main user memcache servers")

    def connect():
        db = database.Connection(options.mysql_host)
        ...

The main() method of your application does not need to be aware of all of
the options used throughout your program; they are all automatically loaded
when the modules are loaded. Your main() method can parse the command line
or parse a config file with::

    import tornado.options
    tornado.options.parse_config_file("/etc/server.conf")
    tornado.options.parse_command_line()

Command line formats are what you would expect ("--myoption=myvalue").
Config files are just Python files. Global names become options, e.g.::

    myoption = "myvalue"
    myotheroption = "myothervalue"

We support datetimes, timedeltas, ints, and floats (just pass a 'type'
kwarg to define). We also accept multi-value options. See the documentation
for define() below.
"""

from __future__ import absolute_import, division, with_statement

import datetime
import logging
import logging.handlers
import re
import sys
import os
import time
import textwrap

from tornado.escape import _unicode

# For pretty log messages, if available
try:
    import curses
except ImportError:
    curses = None


def define(name, default=None, type=None, help=None, metavar=None,
           multiple=False, group=None):
    """Defines a new command line option.

    If type is given (one of str, float, int, datetime, or timedelta)
    or can be inferred from the default, we parse the command line
    arguments based on the given type. If multiple is True, we accept
    comma-separated values, and the option value is always a list.

    For multi-value integers, we also accept the syntax x:y, which
    turns into range(x, y) - very useful for long integer ranges.

    help and metavar are used to construct the automatically generated
    command line help string. The help message is formatted like::

       --name=METAVAR      help string

    group is used to group the defined options in logical groups. By default,
    command line options are grouped by the defined file.

    Command line option names must be unique globally. They can be parsed
    from the command line with parse_command_line() or parsed from a
    config file with parse_config_file.
    """
    if name in options:
        raise Error("Option %r already defined in %s", name,
                    options[name].file_name)
    frame = sys._getframe(0)
    options_file = frame.f_code.co_filename
    file_name = frame.f_back.f_code.co_filename
    if file_name == options_file:
        file_name = ""
    if type is None:
        if not multiple and default is not None:
            type = default.__class__
        else:
            type = str
    if group:
        group_name = group
    else:
        group_name = file_name
    options[name] = _Option(name, file_name=file_name, default=default,
                            type=type, help=help, metavar=metavar,
                            multiple=multiple, group_name=group_name)


def parse_command_line(args=None):
    """Parses all options given on the command line.

    We return all command line arguments that are not options as a list.
    """
    if args is None:
        args = sys.argv
    remaining = []
    for i in xrange(1, len(args)):
        # All things after the last option are command line arguments
        if not args[i].startswith("-"):
            remaining = args[i:]
            break
        if args[i] == "--":
            remaining = args[i + 1:]
            break
        arg = args[i].lstrip("-")
        name, equals, value = arg.partition("=")
        name = name.replace('-', '_')
        if not name in options:
            print_help()
            raise Error('Unrecognized command line option: %r' % name)
        option = options[name]
        if not equals:
            if option.type == bool:
                value = "true"
            else:
                raise Error('Option %r requires a value' % name)
        option.parse(value)
    if options.help:
        print_help()
        sys.exit(0)

    # Set up log level and pretty console logging by default
    if options.logging != 'none':
        logging.getLogger().setLevel(getattr(logging, options.logging.upper()))
        enable_pretty_logging()

    return remaining


def parse_config_file(path):
    """Parses and loads the Python config file at the given path."""
    config = {}
    execfile(path, config, config)
    for name in config:
        if name in options:
            options[name].set(config[name])


def print_help(file=sys.stdout):
    """Prints all the command line options to stdout."""
    print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
    print >> file, "\nOptions:\n"
    by_group = {}
    for option in options.itervalues():
        by_group.setdefault(option.group_name, []).append(option)

    for filename, o in sorted(by_group.items()):
        if filename:
            print >> file, "\n%s options:\n" % os.path.normpath(filename)
        o.sort(key=lambda option: option.name)
        for option in o:
            prefix = option.name
            if option.metavar:
                prefix += "=" + option.metavar
            description = option.help or ""
            if option.default is not None and option.default != '':
                description += " (default %s)" % option.default
            lines = textwrap.wrap(description, 79 - 35)
            if len(prefix) > 30:
                lines.insert(0, '')
            print >> file, "  --%-30s %s" % (prefix, lines[0])
            for line in lines[1:]:
                print >> file, "%-34s %s" % (' ', line)
    print >> file


class _Options(dict):
    """Our global program options, an dictionary with object-like access."""
    @classmethod
    def instance(cls):
        if not hasattr(cls, "_instance"):
            cls._instance = cls()
        return cls._instance

    def __getattr__(self, name):
        if isinstance(self.get(name), _Option):
            return self[name].value()
        raise AttributeError("Unrecognized option %r" % name)


class _Option(object):<commit_msg>fix bug when no help text given for option - wrap returns empty list<commit_after>#!/usr/bin/env python
#
# Copyright 2009 Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A command line parsing module that lets modules define their own options.

Each module defines its own options, e.g.::

    from tornado.options import define, options

    define("mysql_host", default="127.0.0.1:3306", help="Main user DB")
    define("memcache_hosts", default="127.0.0.1:11011", multiple=True,
           help="Main user memcache servers")

    def connect():
        db = database.Connection(options.mysql_host)
        ...

The main() method of your application does not need to be aware of all of
the options used throughout your program; they are all automatically loaded
when the modules are loaded. Your main() method can parse the command line
or parse a config file with::

    import tornado.options
    tornado.options.parse_config_file("/etc/server.conf")
    tornado.options.parse_command_line()

Command line formats are what you would expect ("--myoption=myvalue").
Config files are just Python files. Global names become options, e.g.::

    myoption = "myvalue"
    myotheroption = "myothervalue"

We support datetimes, timedeltas, ints, and floats (just pass a 'type'
kwarg to define). We also accept multi-value options. See the documentation
for define() below.
"""

from __future__ import absolute_import, division, with_statement

import datetime
import logging
import logging.handlers
import re
import sys
import os
import time
import textwrap

from tornado.escape import _unicode

# For pretty log messages, if available
try:
    import curses
except ImportError:
    curses = None


def define(name, default=None, type=None, help=None, metavar=None,
           multiple=False, group=None):
    """Defines a new command line option.

    If type is given (one of str, float, int, datetime, or timedelta)
    or can be inferred from the default, we parse the command line
    arguments based on the given type. If multiple is True, we accept
    comma-separated values, and the option value is always a list.

    For multi-value integers, we also accept the syntax x:y, which
    turns into range(x, y) - very useful for long integer ranges.

    help and metavar are used to construct the automatically generated
    command line help string. The help message is formatted like::

       --name=METAVAR      help string

    group is used to group the defined options in logical groups. By default,
    command line options are grouped by the defined file.

    Command line option names must be unique globally. They can be parsed
    from the command line with parse_command_line() or parsed from a
    config file with parse_config_file.
    """
    if name in options:
        raise Error("Option %r already defined in %s", name,
                    options[name].file_name)
    frame = sys._getframe(0)
    options_file = frame.f_code.co_filename
    file_name = frame.f_back.f_code.co_filename
    if file_name == options_file:
        file_name = ""
    if type is None:
        if not multiple and default is not None:
            type = default.__class__
        else:
            type = str
    if group:
        group_name = group
    else:
        group_name = file_name
    options[name] = _Option(name, file_name=file_name, default=default,
                            type=type, help=help, metavar=metavar,
                            multiple=multiple, group_name=group_name)


def parse_command_line(args=None):
    """Parses all options given on the command line.

    We return all command line arguments that are not options as a list.
    """
    if args is None:
        args = sys.argv
    remaining = []
    for i in xrange(1, len(args)):
        # All things after the last option are command line arguments
        if not args[i].startswith("-"):
            remaining = args[i:]
            break
        if args[i] == "--":
            remaining = args[i + 1:]
            break
        arg = args[i].lstrip("-")
        name, equals, value = arg.partition("=")
        name = name.replace('-', '_')
        if not name in options:
            print_help()
            raise Error('Unrecognized command line option: %r' % name)
        option = options[name]
        if not equals:
            if option.type == bool:
                value = "true"
            else:
                raise Error('Option %r requires a value' % name)
        option.parse(value)
    if options.help:
        print_help()
        sys.exit(0)

    # Set up log level and pretty console logging by default
    if options.logging != 'none':
        logging.getLogger().setLevel(getattr(logging, options.logging.upper()))
        enable_pretty_logging()

    return remaining


def parse_config_file(path):
    """Parses and loads the Python config file at the given path."""
    config = {}
    execfile(path, config, config)
    for name in config:
        if name in options:
            options[name].set(config[name])


def print_help(file=sys.stdout):
    """Prints all the command line options to stdout."""
    print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
    print >> file, "\nOptions:\n"
    by_group = {}
    for option in options.itervalues():
        by_group.setdefault(option.group_name, []).append(option)

    for filename, o in sorted(by_group.items()):
        if filename:
            print >> file, "\n%s options:\n" % os.path.normpath(filename)
        o.sort(key=lambda option: option.name)
        for option in o:
            prefix = option.name
            if option.metavar:
                prefix += "=" + option.metavar
            description = option.help or ""
            if option.default is not None and option.default != '':
                description += " (default %s)" % option.default
            lines = textwrap.wrap(description, 79 - 35)
            if len(prefix) > 30 or len(lines) == 0:
                lines.insert(0, '')
            print >> file, "  --%-30s %s" % (prefix, lines[0])
            for line in lines[1:]:
                print >> file, "%-34s %s" % (' ', line)
    print >> file


class _Options(dict):
    """Our global program options, an dictionary with object-like access."""
    @classmethod
    def instance(cls):
        if not hasattr(cls, "_instance"):
            cls._instance = cls()
        return cls._instance

    def __getattr__(self, name):
        if isinstance(self.get(name), _Option):
            return self[name].value()
        raise AttributeError("Unrecognized option %r" % name)


class _Option(object):
{'fraction_non_alphanumeric': 0.09385136087998872, 'fraction_numerical': 0.006698632068819631, 'mean_word_length': 2.587907918036934, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279401', 'n_tokens_mistral': 3958, 'n_tokens_neox': 3828, 'n_words': 1492}