<commit_before># -*- coding: utf-8  -*-
"""Miscellaneous helper functions (not wiki-dependent)."""
#
# (C) Pywikibot team, 2008-2015
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, print_function, unicode_literals
__version__ = '$Id$'

import bz2
import collections
import gzip
import inspect
import re
import subprocess
import sys
import threading
import time
import types

from distutils.version import Version
from warnings import warn

PYTHON_VERSION = sys.version_info[:3]
PY2 = (PYTHON_VERSION[0] == 2)

if not PY2:
    import queue as Queue

    StringTypes = basestring = (str,)
    UnicodeType = unicode = str
else:
    import Queue

    StringTypes = types.StringTypes
    UnicodeType = types.UnicodeType

from pywikibot.logging import debug


if PYTHON_VERSION < (3, 5):
    # although deprecated in 3 completely no message was emitted until 3.5
    ArgSpec = inspect.ArgSpec
    getargspec = inspect.getargspec
else:
    ArgSpec = collections.namedtuple('ArgSpec', ['args', 'varargs', 'keywords',
                                                 'defaults'])

    def getargspec(func):
        """Python 3 implementation using inspect.signature."""
        sig = inspect.signature(func)
        args = []
        defaults = []
        varargs = None
        kwargs = None
        for p in sig.parameters.values():
            if p.kind == inspect.Parameter.VAR_POSITIONAL:
                varargs = p.name
            elif p.kind == inspect.Parameter.VAR_KEYWORD:
                kwargs = p.name
            else:
                args += [p.name]
                if p.default != inspect.Parameter.empty:
                    defaults += [p.default]
        if defaults:
            defaults = tuple(defaults)
        else:
            defaults = None
        return ArgSpec(args, varargs, kwargs, defaults)


_logger = 'tools'


class _NotImplementedWarning(RuntimeWarning):

    """Feature that is no longer implemented."""

    pass


class NotImplementedClass(object):

    """No implementation is available."""

    def __init__(self, *args, **kwargs):
        """Constructor."""
        raise NotImplementedError(
            '%s: %s' % (self.__class__.__name__, self.__doc__))


if PYTHON_VERSION < (2, 7):
    try:
        import future.backports.misc
    except ImportError:
        warn("""
pywikibot support of Python 2.6 relies on package future for many features.
Please upgrade to Python 2.7+ or Python 3.3+, or run:
    "pip install future>=0.15.0"
""", RuntimeWarning)
        try:
            from ordereddict import OrderedDict
        except ImportError:
            class OrderedDict(NotImplementedClass):

                """OrderedDict not found."""

                pass

        try:
            from counter import Counter
        except ImportError:
            class Counter(NotImplementedClass):

                """Counter not found."""

                pass
        count = None
    else:
        Counter = future.backports.misc.Counter
        OrderedDict = future.backports.misc.OrderedDict

        try:
            count = future.backports.misc.count
        except AttributeError:
            warn('Please update the "future" package to at least version '
                 '0.15.0 to use its count.', RuntimeWarning, 2)
            count = None
        del future

    if count is None:
        def count(start=0, step=1):
            """Backported C{count} to support keyword arguments and step."""
            while True:
                yield start
                start += step


else:
    from collections import Counter  # noqa ; unused
    from collections import OrderedDict
    from itertools import count  # noqa ; unused


def empty_iterator():
    # http://stackoverflow.com/a/13243870/473890
    """An iterator which does nothing."""
    return
    yield


class UnicodeMixin(object):

    """Mixin class to add __str__ method in Python 2 or 3."""

    if not PY2:
        def __str__(self):
            """Return the unicode representation as the str representation."""
            return self.__unicode__()
    else:
        def __str__(self):
            """Return the str representation of the UTF-8 encoded Unicode."""
            return self.__unicode__().encode('utf8')


# From http://python3porting.com/preparing.html
class ComparableMixin(object):

    """Mixin class to allow comparing to other objects which are comparable."""

    def __lt__(self, other):
        """Compare if self is less than other."""
        return other > self._cmpkey()

    def __le__(self, other):
        """Compare if self is less equals other."""
        return other >= self._cmpkey()

    def __eq__(self, other):
        """Compare if self is equal to other."""
        return other == self._cmpkey()

    def __ge__(self, other):
        """Compare if self is greater equals other."""
        return other <= self._cmpkey()

    def __gt__(self, other):
        """Compare if self is greater than other."""
        return other < self._cmpkey()

    def __ne__(self, other):
        """Compare if self is not equal to other."""
        return other != self._cmpkey()


class DotReadableDict(UnicodeMixin):

    """Parent class of Revision() and FileInfo().

    Provide:
    - __getitem__(), __unicode__() and __repr__().

    """

    def __getitem__(self, key):
        """Give access to class values by key.

        Revision class may also give access to its values by keys
        e.g. revid parameter may be assigned by revision['revid']
        as well as revision.revid. This makes formatting strings with
        % operator easier.

        """
        return getattr(self, key)

    def __unicode__(self):
        """Return string representation."""
        # TODO: This is more efficient if the PY2 test is done during
        # class instantiation, and not inside the method.
        if not PY2:
            return repr(self.__dict__)
        else:
            _content = u', '.join(
                u'{0}: {1}'.format(k, v) for k, v in self.__dict__.items())
            return u'{{{0}}}'.format(_content)

    def __repr__(self):
        """Return a more complete string representation."""
        return repr(self.__dict__)


class FrozenDict(dict):

    """
    Frozen dict, preventing write after initialisation.

    Raises TypeError if write attempted.
    """

    def __init__(self, data=None, error=None):
        """
        Constructor.

        @param data: mapping to freeze
        @type data: mapping
        @param error: error message
        @type error: basestring
        """
        if data:
            args = [data]
        else:
            args = []
        super(FrozenDict, self).__init__(*args)
        self._error = error or 'FrozenDict: not writable'

    def update(self, *args, **kwargs):
        """Prevent updates."""
        raise TypeError(self._error)

    __setitem__ = update


def concat_options(message, line_length, options):
    """Concatenate options."""
    indent = len(message) + 2
    line_length -= indent
    option_msg = u''
    option_line = u''
    for option in options:
        if option_line:
            option_line += ', '
        # +1 for ','
        if len(option_line) + len(option) + 1 > line_length:
            if option_msg:
                option_msg += '\n' + ' ' * indent
            option_msg += option_line[:-1]  # remove space
            option_line = ''
        option_line += option
    if option_line:
        if option_msg:
            option_msg += '\n' + ' ' * indent
        option_msg += option_line
    return u'{0} ({1}):'.format(message, option_msg)


class LazyRegex(object):

    """
    Regex object that obtains and compiles the regex on usage.

    Instances behave like the object created using L{re.compile}.
    """

    def __init__(self, pattern, flags=0):
        """
        Constructor.

        @param pattern: L{re} regex pattern
        @type pattern: str or callable
        @param flags: L{re.compile} flags
        @type flags: int
        """
        self.raw = pattern
        self.flags = flags
        super(LazyRegex, self).__init__()

    @property
    def raw(self):
        """Get raw property."""
        if callable(self._raw):
            self._raw = self._raw()

        return self._raw

    @raw.setter
    def raw(self, value):
        """Set raw property."""
        self._raw = value
        self._compiled = None

    @property
    def flags(self):
        """Get flags property."""
        return self._flags

    @flags.setter
    def flags(self, value):
        """Set flags property."""
        self._flags = value
        self._compiled = None

    def __getattr__(self, attr):
        """Compile the regex and delegate all attribute to the regex."""
        if self._raw:
            if not self._compiled:
                self._compiled = re.compile(self.raw, self.flags)

            if hasattr(self._compiled, attr):
                return getattr(self._compiled, attr)

            raise AttributeError('%s: attr %s not recognised'
                                 % (self.__class__.__name__, attr))
        else:
            raise AttributeError('%s.raw not set' % self.__class__.__name__)


class DeprecatedRegex(LazyRegex):

    """Regex object that issues a deprecation notice."""

    def __init__(self, pattern, flags=0, name=None, instead=None):
        """
        Constructor.

        If name is None, the regex pattern will be used as part of
        the deprecation warning.

        @param name: name of the object that is deprecated
        @type name: str or None
        @param instead: if provided, will be used to specify the replacement
            of the deprecated name
        @type instead: str
        """
        super(DeprecatedRegex, self).__init__(pattern, flags)
        self._name = name or self.raw
        self._instead = instead

    def __getattr__(self, attr):
        """Issue deprecation warning."""
        issue_deprecation_warning(
            self._name, self._instead, 2)
        return super(DeprecatedRegex, self).__getattr__(attr)


def first_lower(string):
    """
    Return a string with the first character uncapitalized.

    Empty strings are supported. The original string is not changed.
    """
    return string[:1].lower() + string[1:]


def first_upper(string):
    """
    Return a string with the first character capitalized.

    Empty strings are supported. The original string is not changed.
    """
    return string[:1].upper() + string[1:]


def normalize_username(username):
    """Normalize the username."""
    if not username:
        return None
    username = re.sub('[_ ]+', ' ', username).strip()
    return first_upper(username)


class MediaWikiVersion(Version):

    """
    Version object to allow comparing 'wmf' versions with normal ones.

    The version mainly consist of digits separated by periods. After that is a
    suffix which may only be 'wmf<number>', 'alpha', 'beta<number>' or
    '-rc.<number>' (the - and . are optional). They are considered from old to
    new in that order with a version number without suffix is considered the
    newest. This secondary difference is stored in an internal _dev_version
    attribute.

    Two versions are equal if their normal version and dev version are equal. A
    version is greater if the normal version or dev version is greater. For
    example:

        1.24 < 1.24.1 < 1.25wmf1 < 1.25alpha < 1.25beta1 < 1.25beta2
        < 1.25-rc-1 < 1.25-rc.2 < 1.25

    Any other suffixes are considered invalid.
    """

    MEDIAWIKI_VERSION = re.compile(
        r'^(\d+(?:\.\d+)+)(wmf(\d+)|alpha|beta(\d+)|-?rc\.?(\d+)|.*)?$')

    @classmethod
    def from_generator(cls, generator):
        """Create instance using the generator string."""
        if not generator.startswith('MediaWiki '):
            raise ValueError('Generator string ({0!r}) must start with '
                             '"MediaWiki "'.format(generator))
        return cls(generator[len('MediaWiki '):])

    def parse(self, vstring):
        """Parse version string."""
        version_match = MediaWikiVersion.MEDIAWIKI_VERSION.match(vstring)
        if not version_match:
            raise ValueError('Invalid version number "{0}"'.format(vstring))
        components = [int(n) for n in version_match.group(1).split('.')]
        # The _dev_version numbering scheme might change. E.g. if a stage
        # between 'alpha' and 'beta' is added, 'beta', 'rc' and stable releases
        # are reassigned (beta=3, rc=4, stable=5).
        if version_match.group(3):  # wmf version
            self._dev_version = (0, int(version_match.group(3)))
        elif version_match.group(4):
            self._dev_version = (2, int(version_match.group(4)))
        elif version_match.group(5):
            self._dev_version = (3, int(version_match.group(5)))
        elif version_match.group(2) == 'alpha':
            self._dev_version = (1, )
        else:
            assert 'wmf' not in version_match.group(2)
            assert 'alpha' not in version_match.group(2)
            assert 'beta' not in version_match.group(2)
            assert 'rc' not in version_match.group(2)
            if version_match.group(2):
                debug('Additional unused version part '
                      '"{0}"'.format(version_match.group(2)),
                      _logger)
            self._dev_version = (4, )
        self.suffix = version_match.group(2) or ''
        self.version = tuple(components)

    def __str__(self):
        """Return version number with optional suffix."""
        return '.'.join(str(v) for v in self.version) + self.suffix

    def _cmp(self, other):
        if isinstance(other, basestring):
            other = MediaWikiVersion(other)

        if self.version > other.version:
            return 1<commit_msg>[FIX] MediaWikiVersion: Accept new wmf style<commit_after># -*- coding: utf-8  -*-
"""Miscellaneous helper functions (not wiki-dependent)."""
#
# (C) Pywikibot team, 2008-2015
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, print_function, unicode_literals
__version__ = '$Id$'

import bz2
import collections
import gzip
import inspect
import re
import subprocess
import sys
import threading
import time
import types

from distutils.version import Version
from warnings import warn

PYTHON_VERSION = sys.version_info[:3]
PY2 = (PYTHON_VERSION[0] == 2)

if not PY2:
    import queue as Queue

    StringTypes = basestring = (str,)
    UnicodeType = unicode = str
else:
    import Queue

    StringTypes = types.StringTypes
    UnicodeType = types.UnicodeType

from pywikibot.logging import debug


if PYTHON_VERSION < (3, 5):
    # although deprecated in 3 completely no message was emitted until 3.5
    ArgSpec = inspect.ArgSpec
    getargspec = inspect.getargspec
else:
    ArgSpec = collections.namedtuple('ArgSpec', ['args', 'varargs', 'keywords',
                                                 'defaults'])

    def getargspec(func):
        """Python 3 implementation using inspect.signature."""
        sig = inspect.signature(func)
        args = []
        defaults = []
        varargs = None
        kwargs = None
        for p in sig.parameters.values():
            if p.kind == inspect.Parameter.VAR_POSITIONAL:
                varargs = p.name
            elif p.kind == inspect.Parameter.VAR_KEYWORD:
                kwargs = p.name
            else:
                args += [p.name]
                if p.default != inspect.Parameter.empty:
                    defaults += [p.default]
        if defaults:
            defaults = tuple(defaults)
        else:
            defaults = None
        return ArgSpec(args, varargs, kwargs, defaults)


_logger = 'tools'


class _NotImplementedWarning(RuntimeWarning):

    """Feature that is no longer implemented."""

    pass


class NotImplementedClass(object):

    """No implementation is available."""

    def __init__(self, *args, **kwargs):
        """Constructor."""
        raise NotImplementedError(
            '%s: %s' % (self.__class__.__name__, self.__doc__))


if PYTHON_VERSION < (2, 7):
    try:
        import future.backports.misc
    except ImportError:
        warn("""
pywikibot support of Python 2.6 relies on package future for many features.
Please upgrade to Python 2.7+ or Python 3.3+, or run:
    "pip install future>=0.15.0"
""", RuntimeWarning)
        try:
            from ordereddict import OrderedDict
        except ImportError:
            class OrderedDict(NotImplementedClass):

                """OrderedDict not found."""

                pass

        try:
            from counter import Counter
        except ImportError:
            class Counter(NotImplementedClass):

                """Counter not found."""

                pass
        count = None
    else:
        Counter = future.backports.misc.Counter
        OrderedDict = future.backports.misc.OrderedDict

        try:
            count = future.backports.misc.count
        except AttributeError:
            warn('Please update the "future" package to at least version '
                 '0.15.0 to use its count.', RuntimeWarning, 2)
            count = None
        del future

    if count is None:
        def count(start=0, step=1):
            """Backported C{count} to support keyword arguments and step."""
            while True:
                yield start
                start += step


else:
    from collections import Counter  # noqa ; unused
    from collections import OrderedDict
    from itertools import count  # noqa ; unused


def empty_iterator():
    # http://stackoverflow.com/a/13243870/473890
    """An iterator which does nothing."""
    return
    yield


class UnicodeMixin(object):

    """Mixin class to add __str__ method in Python 2 or 3."""

    if not PY2:
        def __str__(self):
            """Return the unicode representation as the str representation."""
            return self.__unicode__()
    else:
        def __str__(self):
            """Return the str representation of the UTF-8 encoded Unicode."""
            return self.__unicode__().encode('utf8')


# From http://python3porting.com/preparing.html
class ComparableMixin(object):

    """Mixin class to allow comparing to other objects which are comparable."""

    def __lt__(self, other):
        """Compare if self is less than other."""
        return other > self._cmpkey()

    def __le__(self, other):
        """Compare if self is less equals other."""
        return other >= self._cmpkey()

    def __eq__(self, other):
        """Compare if self is equal to other."""
        return other == self._cmpkey()

    def __ge__(self, other):
        """Compare if self is greater equals other."""
        return other <= self._cmpkey()

    def __gt__(self, other):
        """Compare if self is greater than other."""
        return other < self._cmpkey()

    def __ne__(self, other):
        """Compare if self is not equal to other."""
        return other != self._cmpkey()


class DotReadableDict(UnicodeMixin):

    """Parent class of Revision() and FileInfo().

    Provide:
    - __getitem__(), __unicode__() and __repr__().

    """

    def __getitem__(self, key):
        """Give access to class values by key.

        Revision class may also give access to its values by keys
        e.g. revid parameter may be assigned by revision['revid']
        as well as revision.revid. This makes formatting strings with
        % operator easier.

        """
        return getattr(self, key)

    def __unicode__(self):
        """Return string representation."""
        # TODO: This is more efficient if the PY2 test is done during
        # class instantiation, and not inside the method.
        if not PY2:
            return repr(self.__dict__)
        else:
            _content = u', '.join(
                u'{0}: {1}'.format(k, v) for k, v in self.__dict__.items())
            return u'{{{0}}}'.format(_content)

    def __repr__(self):
        """Return a more complete string representation."""
        return repr(self.__dict__)


class FrozenDict(dict):

    """
    Frozen dict, preventing write after initialisation.

    Raises TypeError if write attempted.
    """

    def __init__(self, data=None, error=None):
        """
        Constructor.

        @param data: mapping to freeze
        @type data: mapping
        @param error: error message
        @type error: basestring
        """
        if data:
            args = [data]
        else:
            args = []
        super(FrozenDict, self).__init__(*args)
        self._error = error or 'FrozenDict: not writable'

    def update(self, *args, **kwargs):
        """Prevent updates."""
        raise TypeError(self._error)

    __setitem__ = update


def concat_options(message, line_length, options):
    """Concatenate options."""
    indent = len(message) + 2
    line_length -= indent
    option_msg = u''
    option_line = u''
    for option in options:
        if option_line:
            option_line += ', '
        # +1 for ','
        if len(option_line) + len(option) + 1 > line_length:
            if option_msg:
                option_msg += '\n' + ' ' * indent
            option_msg += option_line[:-1]  # remove space
            option_line = ''
        option_line += option
    if option_line:
        if option_msg:
            option_msg += '\n' + ' ' * indent
        option_msg += option_line
    return u'{0} ({1}):'.format(message, option_msg)


class LazyRegex(object):

    """
    Regex object that obtains and compiles the regex on usage.

    Instances behave like the object created using L{re.compile}.
    """

    def __init__(self, pattern, flags=0):
        """
        Constructor.

        @param pattern: L{re} regex pattern
        @type pattern: str or callable
        @param flags: L{re.compile} flags
        @type flags: int
        """
        self.raw = pattern
        self.flags = flags
        super(LazyRegex, self).__init__()

    @property
    def raw(self):
        """Get raw property."""
        if callable(self._raw):
            self._raw = self._raw()

        return self._raw

    @raw.setter
    def raw(self, value):
        """Set raw property."""
        self._raw = value
        self._compiled = None

    @property
    def flags(self):
        """Get flags property."""
        return self._flags

    @flags.setter
    def flags(self, value):
        """Set flags property."""
        self._flags = value
        self._compiled = None

    def __getattr__(self, attr):
        """Compile the regex and delegate all attribute to the regex."""
        if self._raw:
            if not self._compiled:
                self._compiled = re.compile(self.raw, self.flags)

            if hasattr(self._compiled, attr):
                return getattr(self._compiled, attr)

            raise AttributeError('%s: attr %s not recognised'
                                 % (self.__class__.__name__, attr))
        else:
            raise AttributeError('%s.raw not set' % self.__class__.__name__)


class DeprecatedRegex(LazyRegex):

    """Regex object that issues a deprecation notice."""

    def __init__(self, pattern, flags=0, name=None, instead=None):
        """
        Constructor.

        If name is None, the regex pattern will be used as part of
        the deprecation warning.

        @param name: name of the object that is deprecated
        @type name: str or None
        @param instead: if provided, will be used to specify the replacement
            of the deprecated name
        @type instead: str
        """
        super(DeprecatedRegex, self).__init__(pattern, flags)
        self._name = name or self.raw
        self._instead = instead

    def __getattr__(self, attr):
        """Issue deprecation warning."""
        issue_deprecation_warning(
            self._name, self._instead, 2)
        return super(DeprecatedRegex, self).__getattr__(attr)


def first_lower(string):
    """
    Return a string with the first character uncapitalized.

    Empty strings are supported. The original string is not changed.
    """
    return string[:1].lower() + string[1:]


def first_upper(string):
    """
    Return a string with the first character capitalized.

    Empty strings are supported. The original string is not changed.
    """
    return string[:1].upper() + string[1:]


def normalize_username(username):
    """Normalize the username."""
    if not username:
        return None
    username = re.sub('[_ ]+', ' ', username).strip()
    return first_upper(username)


class MediaWikiVersion(Version):

    """
    Version object to allow comparing 'wmf' versions with normal ones.

    The version mainly consist of digits separated by periods. After that is a
    suffix which may only be 'wmf<number>', 'alpha', 'beta<number>' or
    '-rc.<number>' (the - and . are optional). They are considered from old to
    new in that order with a version number without suffix is considered the
    newest. This secondary difference is stored in an internal _dev_version
    attribute.

    Two versions are equal if their normal version and dev version are equal. A
    version is greater if the normal version or dev version is greater. For
    example:

        1.24 < 1.24.1 < 1.25wmf1 < 1.25alpha < 1.25beta1 < 1.25beta2
        < 1.25-rc-1 < 1.25-rc.2 < 1.25

    Any other suffixes are considered invalid.
    """

    MEDIAWIKI_VERSION = re.compile(
        r'^(\d+(?:\.\d+)+)(-?wmf\.?(\d+)|alpha|beta(\d+)|-?rc\.?(\d+)|.*)?$')

    @classmethod
    def from_generator(cls, generator):
        """Create instance using the generator string."""
        if not generator.startswith('MediaWiki '):
            raise ValueError('Generator string ({0!r}) must start with '
                             '"MediaWiki "'.format(generator))
        return cls(generator[len('MediaWiki '):])

    def parse(self, vstring):
        """Parse version string."""
        version_match = MediaWikiVersion.MEDIAWIKI_VERSION.match(vstring)
        if not version_match:
            raise ValueError('Invalid version number "{0}"'.format(vstring))
        components = [int(n) for n in version_match.group(1).split('.')]
        # The _dev_version numbering scheme might change. E.g. if a stage
        # between 'alpha' and 'beta' is added, 'beta', 'rc' and stable releases
        # are reassigned (beta=3, rc=4, stable=5).
        if version_match.group(3):  # wmf version
            self._dev_version = (0, int(version_match.group(3)))
        elif version_match.group(4):
            self._dev_version = (2, int(version_match.group(4)))
        elif version_match.group(5):
            self._dev_version = (3, int(version_match.group(5)))
        elif version_match.group(2) == 'alpha':
            self._dev_version = (1, )
        else:
            for handled in ('wmf', 'alpha', 'beta', 'rc'):
                # if any of those pops up here our parser has failed
                assert handled not in version_match.group(2), \
                    'Found "{0}" in "{1}"'.format(handled, version_match.group(2))
            if version_match.group(2):
                debug('Additional unused version part '
                      '"{0}"'.format(version_match.group(2)),
                      _logger)
            self._dev_version = (4, )
        self.suffix = version_match.group(2) or ''
        self.version = tuple(components)

    def __str__(self):
        """Return version number with optional suffix."""
        return '.'.join(str(v) for v in self.version) + self.suffix

    def _cmp(self, other):
        if isinstance(other, basestring):
            other = MediaWikiVersion(other)

        if self.version > other.version:
            return 1
{'fraction_non_alphanumeric': 0.10666714537033713, 'fraction_numerical': 0.00990916597853014, 'mean_word_length': 2.1924355300859597, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3073043', 'n_tokens_mistral': 7935, 'n_tokens_neox': 7504, 'n_words': 2486}