<commit_before>'''
This module provides a dummy parser for pythran annotations.
    * spec_parser reads the specs from a python module and returns them.
'''
from __future__ import division
from pythran.types.conversion import pytype_to_pretty_type

from collections import defaultdict
from itertools import product
import re
import os.path
import ply.lex as lex
import ply.yacc as yacc

from pythran.typing import List, Set, Dict, NDArray, Tuple, Pointer, Fun
from pythran.syntax import PythranSyntaxError
from pythran.config import cfg


def ambiguous_types(ty0, ty1):
    from numpy import complex64, complex128
    from numpy import float32, float64
    from numpy import int8, int16, int32, int64, intp, intc
    from numpy import uint8, uint16, uint32, uint64, uintp, uintc
    try:
        from numpy import complex256, float128
    except ImportError:
        complex256 = complex128
        float128 = float64

    if isinstance(ty0, tuple):
        if len(ty0) != len(ty1):
            return False
        return all(ambiguous_types(t0, t1) for t0, t1 in zip(ty0, ty1))

    ambiguous_float_types = float, float64
    if ty0 in ambiguous_float_types and ty1 in ambiguous_float_types:
        return True

    ambiguous_cplx_types = complex, complex128
    if ty0 in ambiguous_cplx_types and ty1 in ambiguous_cplx_types:
        return True

    ambiguous_int_types = int64, int
    if ty0 in ambiguous_int_types and ty1 in ambiguous_int_types:
        return True

    if type(ty0) is not type(ty1):
        return False

    if not hasattr(ty0, '__args__'):
        return False

    if type(ty0) is NDArray:
        # no ambiguity for dtype
        return ambiguous_types(ty0.__args__[1:], ty1.__args__[1:])
    else:
        return ambiguous_types(ty0.__args__, ty1.__args__)


def istransposed(t):
    if not isinstance(t, NDArray):
        return False
    if len(t.__args__) - 1 != 2:
        return False
    return t.__args__[1] == t.__args__[2] == slice(-1, None, None)


def istransposable(t):
    if not isinstance(t, NDArray):
        return False
    if len(t.__args__) - 1 != 2:
        return False
    return all(s.step == 1 for s in t.__args__[1:])


class Spec(object):
    '''
    Result of spec parsing.

    ``functions'' is a mapping from function name to a tuple of signatures
    ``capsule'' is a mapping from function name to signature
    '''

    def __init__(self, functions, capsules=None):
        self.functions = dict(functions)
        self.capsules = capsules or dict()

        # normalize function signatures
        for fname, signatures in functions.items():
            if not isinstance(signatures, tuple):
                self.functions[fname] = (signatures,)

        #self._expand_specs()

        if not self:
            import logging
            logging.warn("No pythran specification, nothing will be exported")

    def keys(self):
        return list(self.functions.keys()) + list(self.capsules.keys())

    def __bool__(self):
        return bool(self.functions or self.capsules)

    __nonzero__ = __bool__

    def to_docstrings(self, docstrings):
        for func_name, signatures in self.functions.items():
            sigdocs = signatures_to_string(func_name, signatures)
            docstring_prototypes = 'Supported prototypes:\n{}'.format(sigdocs)
            docstring_py = docstrings.get(func_name, '')
            if not docstring_py:
                docstring = docstring_prototypes
            else:
                parts = docstring_py.split('\n\n', 1)
                docstring = parts[0] + '\n\n    ' + docstring_prototypes
                if len(parts) == 2:
                    docstring += '\n\n' + parts[1]
            docstrings[func_name] = docstring


class SpecParser(object):

    """
    A parser that scans a file lurking for lines such as the one below.

    It then generates a pythran-compatible signature to inject into compile.
#pythran export a((float,(int, uint8),str list) list list)
#pythran export a(str)
#pythran export a( (str,str), int, int16 list list)
#pythran export a( {str} )
"""

    # lex part
    dtypes = {
        'bool': 'BOOL',
        'byte': 'BYTE',
        'complex': 'COMPLEX',
        'int': 'INT',
        'float': 'FLOAT',
        'uint8': 'UINT8',
        'uint16': 'UINT16',
        'uint32': 'UINT32',
        'uint64': 'UINT64',
        'uintc': 'UINTC',
        'uintp': 'UINTP',
        'int8': 'INT8',
        'int16': 'INT16',
        'int32': 'INT32',
        'int64': 'INT64',
        'intc': 'INTC',
        'intp': 'INTP',
        'float32': 'FLOAT32',
        'float64': 'FLOAT64',
        'float128': 'FLOAT128',
        'complex64': 'COMPLEX64',
        'complex128': 'COMPLEX128',
        'complex256': 'COMPLEX256',
        }

    reserved = {
        '#pythran': 'PYTHRAN',
        'export': 'EXPORT',
        'order': 'ORDER',
        'capsule': 'CAPSULE',
        'or': 'OR',
        'list': 'LIST',
        'set': 'SET',
        'dict': 'DICT',
        'slice': 'SLICE',
        'str': 'STR',
        'None': 'NONE',
        }
    reserved.update(dtypes)

    tokens = ('IDENTIFIER', 'NUM', 'COLUMN', 'LPAREN', 'RPAREN', 'CRAP', 'OPT',
              'LARRAY', 'RARRAY', 'STAR', 'COMMA') + tuple(reserved.values())

    crap = [tok for tok in tokens if tok != 'PYTHRAN']
    some_crap = [tok for tok in crap if tok not in ('LPAREN', 'COMMA')]

    # token <> regexp binding
    t_CRAP = r'[^,:\(\)\[\]*?0-9]'
    t_COMMA = r','
    t_COLUMN = r':'
    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_RARRAY = r'\]'
    t_LARRAY = r'\['
    t_STAR = r'\*'
    t_OPT = r'\?'
    t_NUM = r'[1-9][0-9]*'

    precedence = (
        ('left', 'OR'),
        ('left', 'LIST', 'DICT', 'SET'),
    )

    def t_IDENTIFER(self, t):
        r'\#?[a-zA-Z_][a-zA-Z_0-9]*'
        t.type = SpecParser.reserved.get(t.value, 'IDENTIFIER')
        return t

    # skipped characters
    t_ignore = ' \t\r'

    # error handling
    def t_error(self, t):
        t.lexer.skip(1)

    # Define a rule so we can track line numbers
    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    # yacc part

    def p_exports(self, p):
        '''exports :
                   | PYTHRAN EXPORT export_list opt_craps exports
                   | PYTHRAN EXPORT CAPSULE export_list opt_craps exports'''
        if len(p) > 1:
            target = self.exports if len(p) == 6 else self.native_exports
            for key, val in p[len(p)-3]:
                target[key] += val

    def p_export_list(self, p):
        '''export_list : export
                  | export_list COMMA export'''
        p[0] = (p[1],) if len(p) == 2 else (p[1] + (p[3],))

    def p_export(self, p):
        '''export : IDENTIFIER LPAREN opt_param_types RPAREN
                  | IDENTIFIER
                  | EXPORT LPAREN opt_param_types RPAREN
                  | ORDER LPAREN opt_param_types RPAREN'''
        # unlikely case: the IDENTIFIER is an otherwise reserved name
        if len(p) > 2:
            sigs = p[3] or ((),)
        else:
            sigs = ()
        p[0] = p[1], sigs

    def p_opt_craps(self, p):
        '''opt_craps :
                     | some_crap opt_all_craps'''

    def p_opt_all_craps(self, p):
        '''opt_all_craps :
                     | crap opt_all_craps'''

    def p_crap(self, p):
        'crap : '

    def p_some_crap(self, p):
        'some_crap : '

    p_crap.__doc__ += '\n| '.join(crap)
    p_some_crap.__doc__ += '\n| '.join(some_crap)

    def p_dtype(self, p):
        'dtype : '
        import numpy
        p[0] = eval(p[1], numpy.__dict__),

    p_dtype.__doc__ += '\n| '.join(dtypes.values())

    def p_opt_param_types(self, p):
        '''opt_param_types :
                     | param_types'''
        p[0] = p[1] if len(p) == 2 else tuple()

    def p_opt_types(self, p):
        '''opt_types :
                     | types'''
        p[0] = p[1] if len(p) == 2 else tuple()

    def p_param_types(self, p):
        '''param_types : type
                       | type OPT
                       | type COMMA param_types
                       | type OPT COMMA default_types'''
        if len(p) == 2:
            p[0] = tuple((t,) for t in p[1])
        elif len(p) == 3:
            p[0] = tuple((t,) for t in p[1]) + ((),)
        elif len(p) == 4:
            p[0] = tuple((t,) + ts for t in p[1] for ts in p[3])
        else:
            non_defaults = [t for t in p[4] if len(t) == len(p[4][0])]<commit_msg>Minor update to docstring generator with ply<commit_after>'''
This module provides a dummy parser for pythran annotations.
    * spec_parser reads the specs from a python module and returns them.
'''
from __future__ import division
from pythran.types.conversion import pytype_to_pretty_type

from collections import defaultdict
from itertools import product
import re
import os.path
import ply.lex as lex
import ply.yacc as yacc

from pythran.typing import List, Set, Dict, NDArray, Tuple, Pointer, Fun
from pythran.syntax import PythranSyntaxError
from pythran.config import cfg


def ambiguous_types(ty0, ty1):
    from numpy import complex64, complex128
    from numpy import float32, float64
    from numpy import int8, int16, int32, int64, intp, intc
    from numpy import uint8, uint16, uint32, uint64, uintp, uintc
    try:
        from numpy import complex256, float128
    except ImportError:
        complex256 = complex128
        float128 = float64

    if isinstance(ty0, tuple):
        if len(ty0) != len(ty1):
            return False
        return all(ambiguous_types(t0, t1) for t0, t1 in zip(ty0, ty1))

    ambiguous_float_types = float, float64
    if ty0 in ambiguous_float_types and ty1 in ambiguous_float_types:
        return True

    ambiguous_cplx_types = complex, complex128
    if ty0 in ambiguous_cplx_types and ty1 in ambiguous_cplx_types:
        return True

    ambiguous_int_types = int64, int
    if ty0 in ambiguous_int_types and ty1 in ambiguous_int_types:
        return True

    if type(ty0) is not type(ty1):
        return False

    if not hasattr(ty0, '__args__'):
        return False

    if type(ty0) is NDArray:
        # no ambiguity for dtype
        return ambiguous_types(ty0.__args__[1:], ty1.__args__[1:])
    else:
        return ambiguous_types(ty0.__args__, ty1.__args__)


def istransposed(t):
    if not isinstance(t, NDArray):
        return False
    if len(t.__args__) - 1 != 2:
        return False
    return t.__args__[1] == t.__args__[2] == slice(-1, None, None)


def istransposable(t):
    if not isinstance(t, NDArray):
        return False
    if len(t.__args__) - 1 != 2:
        return False
    return all(s.step == 1 for s in t.__args__[1:])


class Spec(object):
    '''
    Result of spec parsing.

    ``functions'' is a mapping from function name to a tuple of signatures
    ``capsule'' is a mapping from function name to signature
    '''

    def __init__(self, functions, capsules=None):
        self.functions = dict(functions)
        self.capsules = capsules or dict()

        # normalize function signatures
        for fname, signatures in functions.items():
            if not isinstance(signatures, tuple):
                self.functions[fname] = (signatures,)

        #self._expand_specs()

        if not self:
            import logging
            logging.warn("No pythran specification, nothing will be exported")

    def keys(self):
        return list(self.functions.keys()) + list(self.capsules.keys())

    def __bool__(self):
        return bool(self.functions or self.capsules)

    __nonzero__ = __bool__

    def to_docstrings(self, docstrings):
        for func_name, signatures in self.functions.items():
            sigdocs = signatures_to_string(func_name, signatures)
            docstring_prototypes = 'Supported prototypes:\n{}'.format(sigdocs)
            docstring_py = docstrings.get(func_name, '')
            if not docstring_py:
                docstring = docstring_prototypes
            else:
                parts = docstring_py.split('\n\n', 1)
                docstring = parts[0] + '\n\n    ' + docstring_prototypes
                if len(parts) == 2:
                    docstring += '\n\n' + parts[1]
            docstrings[func_name] = docstring


class SpecParser(object):

    """
    A parser that scans a file lurking for lines such as the one below.

    It then generates a pythran-compatible signature to inject into compile.
#pythran export a((float,(int, uint8),str list) list list)
#pythran export a(str)
#pythran export a( (str,str), int, int16 list list)
#pythran export a( {str} )
"""

    # lex part
    dtypes = {
        'bool': 'BOOL',
        'byte': 'BYTE',
        'complex': 'COMPLEX',
        'int': 'INT',
        'float': 'FLOAT',
        'uint8': 'UINT8',
        'uint16': 'UINT16',
        'uint32': 'UINT32',
        'uint64': 'UINT64',
        'uintc': 'UINTC',
        'uintp': 'UINTP',
        'int8': 'INT8',
        'int16': 'INT16',
        'int32': 'INT32',
        'int64': 'INT64',
        'intc': 'INTC',
        'intp': 'INTP',
        'float32': 'FLOAT32',
        'float64': 'FLOAT64',
        'float128': 'FLOAT128',
        'complex64': 'COMPLEX64',
        'complex128': 'COMPLEX128',
        'complex256': 'COMPLEX256',
        }

    reserved = {
        '#pythran': 'PYTHRAN',
        'export': 'EXPORT',
        'order': 'ORDER',
        'capsule': 'CAPSULE',
        'or': 'OR',
        'list': 'LIST',
        'set': 'SET',
        'dict': 'DICT',
        'slice': 'SLICE',
        'str': 'STR',
        'None': 'NONE',
        }
    reserved.update(dtypes)

    tokens = ('IDENTIFIER', 'NUM', 'COLUMN', 'LPAREN', 'RPAREN', 'CRAP', 'OPT',
              'LARRAY', 'RARRAY', 'STAR', 'COMMA') + tuple(reserved.values())

    crap = [tok for tok in tokens if tok != 'PYTHRAN']
    some_crap = [tok for tok in crap if tok not in ('LPAREN', 'COMMA')]

    # token <> regexp binding
    t_CRAP = r'[^,:\(\)\[\]*?0-9]'
    t_COMMA = r','
    t_COLUMN = r':'
    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_RARRAY = r'\]'
    t_LARRAY = r'\['
    t_STAR = r'\*'
    t_OPT = r'\?'
    t_NUM = r'[1-9][0-9]*'

    precedence = (
        ('left', 'OR'),
        ('left', 'LIST', 'DICT', 'SET'),
    )

    def t_IDENTIFER(self, t):
        r'\#?[a-zA-Z_][a-zA-Z_0-9]*'
        t.type = SpecParser.reserved.get(t.value, 'IDENTIFIER')
        return t

    # skipped characters
    t_ignore = ' \t\r'

    # error handling
    def t_error(self, t):
        t.lexer.skip(1)

    # Define a rule so we can track line numbers
    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    # yacc part

    def p_exports(self, p):
        '''exports :
                   | PYTHRAN EXPORT export_list opt_craps exports
                   | PYTHRAN EXPORT CAPSULE export_list opt_craps exports'''
        if len(p) > 1:
            target = self.exports if len(p) == 6 else self.native_exports
            for key, val in p[len(p)-3]:
                target[key] += val

    def p_export_list(self, p):
        '''export_list : export
                  | export_list COMMA export'''
        p[0] = (p[1],) if len(p) == 2 else (p[1] + (p[3],))

    def p_export(self, p):
        '''export : IDENTIFIER LPAREN opt_param_types RPAREN
                  | IDENTIFIER
                  | EXPORT LPAREN opt_param_types RPAREN
                  | ORDER LPAREN opt_param_types RPAREN'''
        # unlikely case: the IDENTIFIER is an otherwise reserved name
        if len(p) > 2:
            sigs = p[3] or ((),)
        else:
            sigs = ()
        p[0] = p[1], sigs

    def p_opt_craps(self, p):
        '''opt_craps :
                     | some_crap opt_all_craps'''

    def p_opt_all_craps(self, p):
        '''opt_all_craps :
                     | crap opt_all_craps'''

    def p_crap(self, p):
        pass

    p_crap.__doc__ = 'crap : ' + '\n| '.join(crap)

    def p_some_crap(self, p):
        pass

    p_some_crap.__doc__ = 'some_crap : ' + '\n| '.join(some_crap)

    def p_dtype(self, p):
        'dtype : '
        import numpy
        p[0] = eval(p[1], numpy.__dict__),

    p_dtype.__doc__ += '\n| '.join(dtypes.values())

    def p_opt_param_types(self, p):
        '''opt_param_types :
                     | param_types'''
        p[0] = p[1] if len(p) == 2 else tuple()

    def p_opt_types(self, p):
        '''opt_types :
                     | types'''
        p[0] = p[1] if len(p) == 2 else tuple()

    def p_param_types(self, p):
        '''param_types : type
                       | type OPT
                       | type COMMA param_types
                       | type OPT COMMA default_types'''
        if len(p) == 2:
            p[0] = tuple((t,) for t in p[1])
        elif len(p) == 3:
            p[0] = tuple((t,) for t in p[1]) + ((),)
        elif len(p) == 4:
            p[0] = tuple((t,) + ts for t in p[1] for ts in p[3])
        else:
            non_defaults = [t for t in p[4] if len(t) == len(p[4][0])]
{'fraction_non_alphanumeric': 0.12974051896207583, 'fraction_numerical': 0.022308324527415758, 'mean_word_length': 2.0927741466957155, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 6, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '179467', 'n_tokens_mistral': 5958, 'n_tokens_neox': 5573, 'n_words': 1553}