<commit_before># cython: language_level = 2
#
#   Code output module
#

from __future__ import absolute_import

import cython
cython.declare(os=object, re=object, operator=object,
               Naming=object, Options=object, StringEncoding=object,
               Utils=object, SourceDescriptor=object, StringIOTree=object,
               DebugFlags=object, basestring=object)

import os
import re
import sys
from string import Template
import operator
import textwrap

try:
    import hashlib
except ImportError:
    import md5 as hashlib

from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree

try:
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring

KEYWORDS_MUST_BE_BYTES = sys.version_info < (2,7)


non_portable_builtins_map = {
    # builtins that have different names in different Python versions
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
    'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'),
    }

basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
    'PyTypeObject' : 'PyHeapTypeObject',
    }

uncachable_builtins = [
    # builtin names that cannot be cached because they may or may not
    # be available at import time
    'WindowsError',
    ]

modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get

is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match


def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")


class UtilityCodeBase(object):
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####

        [proto declarations]

        ##### MyUtility.init #####

        [code run at module initialization]

        ##### MyUtility #####
        #@requires: MyOtherUtility
        #@substitute: naming

        [definitions]

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """

    is_cython_utility = False
    requires = None
    _utility_cache = {}

    @classmethod
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
        if utility is None:
            return

        code = '\n'.join(lines)
        if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
            del tags['substitute']
            try:
                code = Template(code).substitute(vars(Naming))
            except (KeyError, ValueError), e:
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code

        if type == 'proto':
            utility[0] = code
        elif type == 'impl':
            utility[1] = code
        else:
            all_tags = utility[2]
            if KEYWORDS_MUST_BE_BYTES:
                type = type.encode('ASCII')
            all_tags[type] = code

        if tags:
            all_tags = utility[2]
            for name, values in tags.items():
                if KEYWORDS_MUST_BE_BYTES:
                    name = name.encode('ASCII')
                all_tags.setdefault(name, set()).update(values)

    @classmethod
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

        filename = os.path.join(get_utility_dir(), path)
        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
            replace_comments = re.compile(r'^\s*#.*').sub
        else:
            comment = '/'
            replace_comments = re.compile(r'^\s*//.*|^\s*/\*[^*]*\*/').sub
        match_special = re.compile(
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)'
                ) % {'C':comment}).match
        match_type = re.compile('(.+)[.](proto|impl|init|cleanup)$').match

        f = Utils.open_source_file(filename, encoding='UTF-8')
        try:
            all_lines = f.readlines()
        finally:
            f.close()

        utilities = {}
        lines = []
        tags = {}
        utility = type = None
        begin_lineno = 0

        for lineno, line in enumerate(all_lines):
            m = match_special(line)
            if m:
                if m.group('name'):
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1<commit_msg>discard all 1-line /*comments*/ from utility C code (not just at the beginning of a line)<commit_after># cython: language_level = 2
#
#   Code output module
#

from __future__ import absolute_import

import cython
cython.declare(os=object, re=object, operator=object,
               Naming=object, Options=object, StringEncoding=object,
               Utils=object, SourceDescriptor=object, StringIOTree=object,
               DebugFlags=object, basestring=object)

import os
import re
import sys
from string import Template
import operator
import textwrap

try:
    import hashlib
except ImportError:
    import md5 as hashlib

from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree

try:
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring

KEYWORDS_MUST_BE_BYTES = sys.version_info < (2,7)


non_portable_builtins_map = {
    # builtins that have different names in different Python versions
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
    'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'),
    }

basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
    'PyTypeObject' : 'PyHeapTypeObject',
    }

uncachable_builtins = [
    # builtin names that cannot be cached because they may or may not
    # be available at import time
    'WindowsError',
    ]

modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get

is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match


def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")


class UtilityCodeBase(object):
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####

        [proto declarations]

        ##### MyUtility.init #####

        [code run at module initialization]

        ##### MyUtility #####
        #@requires: MyOtherUtility
        #@substitute: naming

        [definitions]

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """

    is_cython_utility = False
    requires = None
    _utility_cache = {}

    @classmethod
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
        if utility is None:
            return

        code = '\n'.join(lines)
        if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
            del tags['substitute']
            try:
                code = Template(code).substitute(vars(Naming))
            except (KeyError, ValueError), e:
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code

        if type == 'proto':
            utility[0] = code
        elif type == 'impl':
            utility[1] = code
        else:
            all_tags = utility[2]
            if KEYWORDS_MUST_BE_BYTES:
                type = type.encode('ASCII')
            all_tags[type] = code

        if tags:
            all_tags = utility[2]
            for name, values in tags.items():
                if KEYWORDS_MUST_BE_BYTES:
                    name = name.encode('ASCII')
                all_tags.setdefault(name, set()).update(values)

    @classmethod
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

        filename = os.path.join(get_utility_dir(), path)
        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
            replace_comments = re.compile(r'^\s*#.*').sub
        else:
            comment = '/'
            replace_comments = re.compile(r'^\s*//.*|/\*[^*]*\*/').sub
        match_special = re.compile(
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)'
                ) % {'C':comment}).match
        match_type = re.compile('(.+)[.](proto|impl|init|cleanup)$').match

        f = Utils.open_source_file(filename, encoding='UTF-8')
        try:
            all_lines = f.readlines()
        finally:
            f.close()

        utilities = {}
        lines = []
        tags = {}
        utility = type = None
        begin_lineno = 0

        for lineno, line in enumerate(all_lines):
            m = match_special(line)
            if m:
                if m.group('name'):
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1
{'fraction_non_alphanumeric': 0.12145454545454545, 'fraction_numerical': 0.007, 'mean_word_length': 2.319553409776705, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5751201', 'n_tokens_mistral': 3469, 'n_tokens_neox': 3333, 'n_words': 890}