<commit_before>#######################################################################
# This file is part of PyBlosxom.
#
# Copyright (c) 2003-2006 <NAME>
# Copyright (c) 2003-2010 <NAME>
#
# PyBlosxom is distributed under the MIT license.  See the file
# LICENSE for distribution details.
#######################################################################
"""Utility module for functions that are useful to PyBlosxom and plugins.
"""

import sgmllib
import re
import os
import time
import os.path
import stat
import sys
import locale
import urllib
import inspect

# Pyblosxom imports
from Pyblosxom import plugin_utils

# Note: month names tend to differ with locale

# month name (Jan) to number (1)
month2num = None
# month number (1) to name (Jan)
num2month = None
# list of all month numbers and names
MONTHS    = None

# regular expression for detection and substituion of variables.
_VAR_REGEXP = re.compile(r"""
    (?<!\\)   # if the $ is escaped, then this isn't a variable
    \$        # variables start with a $
    (
        (?:\w|\-|::\w)+       # word char, - or :: followed by a word char
        (?:
            \(                # an open paren
            .*?               # followed by non-greedy bunch of stuff
            (?<!\\)\)         # with an end paren that's not escaped
        )?    # 0 or 1 of these ( ... ) blocks
    |
        \(
        (?:\w|\-|::\w)+       # word char, - or :: followed by a word char
        (?:
            \(                # an open paren
            .*?               # followed by non-greedy bunch of stuff
            (?<!\\)\)         # with an end paren that's not escaped
        )?    # 0 or 1 of these ( ... ) blocks
        \)
    )
    """, re.VERBOSE)

# reference to the pyblosxom config dict
_config = {}

def initialize(config):
    """Initializes the tools module.

    This gives the module a chance to use configuration from the
    pyblosxom config.py file.

    This should be called from ``Pyblosxom.pyblosxom.PyBlosxom.initialize``.
    """
    global _config
    _config = config

    # Month names tend to differ with locale
    global month2num

    try:
        month2num = {'nil' : '00',
                     locale.nl_langinfo(locale.ABMON_1) : '01',
                     locale.nl_langinfo(locale.ABMON_2) : '02',
                     locale.nl_langinfo(locale.ABMON_3) : '03',
                     locale.nl_langinfo(locale.ABMON_4) : '04',
                     locale.nl_langinfo(locale.ABMON_5) : '05',
                     locale.nl_langinfo(locale.ABMON_6) : '06',
                     locale.nl_langinfo(locale.ABMON_7) : '07',
                     locale.nl_langinfo(locale.ABMON_8) : '08',
                     locale.nl_langinfo(locale.ABMON_9) : '09',
                     locale.nl_langinfo(locale.ABMON_10) : '10',
                     locale.nl_langinfo(locale.ABMON_11) : '11',
                     locale.nl_langinfo(locale.ABMON_12) : '12'}

    except AttributeError:
        # Windows doesn't have nl_langinfo, so we use one that
        # only return English.
        # FIXME - need a better hack for this issue.
        month2num = {'nil': '00',
                     "Jan": '01',
                     "Feb": '02',
                     "Mar": '03',
                     "Apr": '04',
                     "May": '05',
                     "Jun": '06',
                     "Jul": '07',
                     "Aug": '08',
                     "Sep": '09',
                     "Oct": '10',
                     "Nov": '11',
                     "Dec": '12'}

    # This is not python 2.1 compatible (Nifty though)
    # num2month = dict(zip(month2num.itervalues(), month2num))
    global num2month
    num2month = {}
    for month_abbr, month_num in month2num.items():
        num2month[month_num] = month_abbr
        num2month[int(month_num)] = month_abbr

    # all the valid month possibilities
    global MONTHS
    MONTHS = num2month.keys() + month2num.keys()

def deprecated_function(func):
    def _deprecated_function(*args, **kwargs):
        return func(*args, **kwargs)

    _deprecated_function.__doc__ = ("DEPRECATED.  Use %s instead." % 
                                    func.__name__)
    _deprecated_function.__dict__.update(func.__dict__)
    return _deprecated_function

class ConfigSyntaxErrorException(Exception):
    """Thrown when ``convert_configini_values`` encounters a syntax
    error.
    """
    pass

def convert_configini_values(configini):
    """Takes a dict containing config.ini style keys and values, converts
    the values, and returns a new config dict.

    :param confini: dict containing the config.ini style keys and values

    :raises ConfigSyntaxErrorException: when there's a syntax error
    """
    def s_or_i(text):
        """Takes a string and if it begins with " or ' and ends with
        " or ', then it returns the string.  If it's an int, returns
        the int.  Otherwise it returns the text.
        """
        if text.startswith('"'):
            if text.endswith('"'):
                return text[1:-1]
            raise ConfigSyntaxErrorException("config syntax error: "
                                             "string '%s' missing end \"" %
                                             text)
        elif text.startswith("'"):
            if text.endswith("'"):
                return text[1:-1]
            raise ConfigSyntaxErrorException("config syntax error: "
                                             "string '%s' missing end '" %
                                             text)
        elif text.isdigit():
            return int(text)
        return text

    config = {}
    for key, value in configini.items():
        # in configini.items, we pick up a local_config which seems
        # to be a copy of what's in configini.items--puzzling.
        if isinstance(value, dict):
            continue

        value = value.strip()
        if value.startswith("["):
            if value.endswith("]"):
                value2 = value[1:-1].strip().split(",")
                if len(value2) == 1 and value2[0] == "":
                    # handle the foo = [] case
                    config[key] = []
                else:
                    config[key] = [s_or_i(s.strip()) for s in value2]
            else:
                raise ConfigSyntaxErrorException("config syntax error: "
                                                 "list '%s' missing end ]" %
                                                 value)
        else:
            config[key] = s_or_i(value)

    return config

def escape_text(s):
    """Takes in a string and converts:

    * ``&`` to ``&amp;``
    * ``>`` to ``&gt;``
    * ``<`` to ``&lt;``
    * ``\"`` to ``&quot;``
    * ``'`` to ``&#x27;``
    * ``/`` to ``&#x2F;``

    Note: if ``s`` is ``None``, then we return ``None``.

    >>> escape_text(None)
    None
    >>> escape_text("")
    ""
    >>> escape_text("a'b")
    "a&#x27;b"<commit_msg>Fixed configini parsing where the beginning bit is missing, but the end bit is there.  Examples that weren't parsing right: b', b", and b].<commit_after>#######################################################################
# This file is part of PyBlosxom.
#
# Copyright (c) 2003-2006 <NAME>
# Copyright (c) 2003-2010 <NAME>
#
# PyBlosxom is distributed under the MIT license.  See the file
# LICENSE for distribution details.
#######################################################################
"""Utility module for functions that are useful to PyBlosxom and plugins.
"""

import sgmllib
import re
import os
import time
import os.path
import stat
import sys
import locale
import urllib
import inspect

# Pyblosxom imports
from Pyblosxom import plugin_utils

# Note: month names tend to differ with locale

# month name (Jan) to number (1)
month2num = None
# month number (1) to name (Jan)
num2month = None
# list of all month numbers and names
MONTHS    = None

# regular expression for detection and substituion of variables.
_VAR_REGEXP = re.compile(r"""
    (?<!\\)   # if the $ is escaped, then this isn't a variable
    \$        # variables start with a $
    (
        (?:\w|\-|::\w)+       # word char, - or :: followed by a word char
        (?:
            \(                # an open paren
            .*?               # followed by non-greedy bunch of stuff
            (?<!\\)\)         # with an end paren that's not escaped
        )?    # 0 or 1 of these ( ... ) blocks
    |
        \(
        (?:\w|\-|::\w)+       # word char, - or :: followed by a word char
        (?:
            \(                # an open paren
            .*?               # followed by non-greedy bunch of stuff
            (?<!\\)\)         # with an end paren that's not escaped
        )?    # 0 or 1 of these ( ... ) blocks
        \)
    )
    """, re.VERBOSE)

# reference to the pyblosxom config dict
_config = {}

def initialize(config):
    """Initializes the tools module.

    This gives the module a chance to use configuration from the
    pyblosxom config.py file.

    This should be called from ``Pyblosxom.pyblosxom.PyBlosxom.initialize``.
    """
    global _config
    _config = config

    # Month names tend to differ with locale
    global month2num

    try:
        month2num = {'nil' : '00',
                     locale.nl_langinfo(locale.ABMON_1) : '01',
                     locale.nl_langinfo(locale.ABMON_2) : '02',
                     locale.nl_langinfo(locale.ABMON_3) : '03',
                     locale.nl_langinfo(locale.ABMON_4) : '04',
                     locale.nl_langinfo(locale.ABMON_5) : '05',
                     locale.nl_langinfo(locale.ABMON_6) : '06',
                     locale.nl_langinfo(locale.ABMON_7) : '07',
                     locale.nl_langinfo(locale.ABMON_8) : '08',
                     locale.nl_langinfo(locale.ABMON_9) : '09',
                     locale.nl_langinfo(locale.ABMON_10) : '10',
                     locale.nl_langinfo(locale.ABMON_11) : '11',
                     locale.nl_langinfo(locale.ABMON_12) : '12'}

    except AttributeError:
        # Windows doesn't have nl_langinfo, so we use one that
        # only return English.
        # FIXME - need a better hack for this issue.
        month2num = {'nil': '00',
                     "Jan": '01',
                     "Feb": '02',
                     "Mar": '03',
                     "Apr": '04',
                     "May": '05',
                     "Jun": '06',
                     "Jul": '07',
                     "Aug": '08',
                     "Sep": '09',
                     "Oct": '10',
                     "Nov": '11',
                     "Dec": '12'}

    # This is not python 2.1 compatible (Nifty though)
    # num2month = dict(zip(month2num.itervalues(), month2num))
    global num2month
    num2month = {}
    for month_abbr, month_num in month2num.items():
        num2month[month_num] = month_abbr
        num2month[int(month_num)] = month_abbr

    # all the valid month possibilities
    global MONTHS
    MONTHS = num2month.keys() + month2num.keys()

def deprecated_function(func):
    def _deprecated_function(*args, **kwargs):
        return func(*args, **kwargs)

    _deprecated_function.__doc__ = ("DEPRECATED.  Use %s instead." % 
                                    func.__name__)
    _deprecated_function.__dict__.update(func.__dict__)
    return _deprecated_function

class ConfigSyntaxErrorException(Exception):
    """Thrown when ``convert_configini_values`` encounters a syntax
    error.
    """
    pass

def convert_configini_values(configini):
    """Takes a dict containing config.ini style keys and values, converts
    the values, and returns a new config dict.

    :param confini: dict containing the config.ini style keys and values

    :raises ConfigSyntaxErrorException: when there's a syntax error
    """
    def s_or_i(text):
        """
        Takes a string and if it begins with \" or \' and ends with
        \" or \', then it returns the string.  If it's an int, returns
        the int.  Otherwise it returns the text.
        """
        text = text.strip()
        if (((text.startswith('"') and not text.endswith('"'))
             or (not text.startswith('"') and text.endswith('"')))):
            raise ConfigSyntaxErrorException(
                "config syntax error: string '%s' missing start or end \"" %
                text)
        elif (((text.startswith("'") and not text.endswith("'"))
               or (not text.startswith("'") and text.endswith("'")))):
            raise ConfigSyntaxErrorException(
                "config syntax error: string '%s' missing start or end '" %
                text)
        elif text.startswith('"') and text.endswith('"'):
            return text[1:-1]
        elif text.startswith("'") and text.endswith("'"):
            return text[1:-1]
        elif text.isdigit():
            return int(text)
        return text

    config = {}
    for key, value in configini.items():
        # in configini.items, we pick up a local_config which seems
        # to be a copy of what's in configini.items--puzzling.
        if isinstance(value, dict):
            continue

        value = value.strip()
        if (((value.startswith("[") and not value.endswith("]"))
             or (not value.startswith("[") and value.endswith("]")))):
            raise ConfigSyntaxErrorException(
                "config syntax error: list '%s' missing [ or ]" %
                value)
        elif value.startswith("[") and value.endswith("]"):
            value2 = value[1:-1].strip().split(",")
            if len(value2) == 1 and value2[0] == "":
                # handle the foo = [] case
                config[key] = []
            else:
                config[key] = [s_or_i(s.strip()) for s in value2]
        else:
            config[key] = s_or_i(value)

    return config

def escape_text(s):
    """Takes in a string and converts:

    * ``&`` to ``&amp;``
    * ``>`` to ``&gt;``
    * ``<`` to ``&lt;``
    * ``\"`` to ``&quot;``
    * ``'`` to ``&#x27;``
    * ``/`` to ``&#x2F;``

    Note: if ``s`` is ``None``, then we return ``None``.

    >>> escape_text(None)
    None
    >>> escape_text("")
    ""
    >>> escape_text("a'b")
    "a&#x27;b"
{'fraction_non_alphanumeric': 0.14193638783802717, 'fraction_numerical': 0.017234131988230348, 'mean_word_length': 1.808380877434586, 'pattern_counts': {'":': 26, '<': 15, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3554191', 'n_tokens_mistral': 4313, 'n_tokens_neox': 4030, 'n_words': 1208}