<commit_before>
from __future__ import absolute_import
import types
import re
import compiler
from compiler import ast

from .algebraic_structures import BasicAlgebra

OR = intern(' or ')
AND = intern(' and ')
NOT = intern('not ')

LT = intern('<')
LE = intern('<=')
GT = intern('>')
GE = intern('>=')
EQ = intern('==')
NE = intern('!=')

BAND = intern('&')
BOR = intern('|')
BXOR = intern('^')
INVERT = intern('~')

POS = intern('+')
NEG = intern('-')
ADD = intern(' + ')
SUB = intern(' - ')
MOD = intern('%')
MUL = intern('*')
DIV = intern('/')
POW = intern('**')

NUMBER = intern('N')
SYMBOL = intern('S')
APPLY = intern('A')
TUPLE = intern('T')
LAMBDA = intern('L')

# XXX: Unimplemented expression parts:
# XXX: LeftShift, RightShift, List*, Subscript, Slice, KeyWord, GetAttr, Ellipsis
# XXX: function calls assume no optional nor *args nor *kwargs, same applies to lambda

boolean_lst = [AND, OR, NOT]
compare_lst = [LT, LE, GT, GE, EQ, NE]
bit_lst = [BAND, BOR, BXOR, INVERT]
arith_lst = [POS, NEG, ADD, SUB, MOD, MUL, DIV, POW]
parentheses_map = {
    OR: [LAMBDA],
    AND: [LAMBDA, OR],
    NOT: [LAMBDA, AND, OR],
    LT: [LAMBDA] + boolean_lst,
    LE: [LAMBDA] + boolean_lst,
    GT: [LAMBDA] + boolean_lst,
    GE: [LAMBDA] + boolean_lst,
    EQ: [LAMBDA] + boolean_lst,
    NE: [LAMBDA] + boolean_lst,
    BOR: [LAMBDA] + compare_lst + boolean_lst,
    BXOR: [LAMBDA, BOR] + compare_lst + boolean_lst,
    BAND: [LAMBDA, BOR, BXOR] + compare_lst + boolean_lst,
    INVERT: [LAMBDA, BOR, BXOR, BAND] + compare_lst + boolean_lst,
    ADD: [LAMBDA] + compare_lst + boolean_lst,
    SUB: [LAMBDA, ADD] + compare_lst + boolean_lst,
    POS: [LAMBDA, ADD, SUB] + compare_lst + boolean_lst,
    NEG: [LAMBDA, ADD, SUB] + compare_lst + boolean_lst,
    MOD: [LAMBDA, ADD, SUB, POS, NEG] + compare_lst + boolean_lst,
    MUL: [LAMBDA, ADD, SUB, POS, NEG, MOD] + compare_lst + boolean_lst,
    DIV: [LAMBDA, ADD, SUB, POS, NEG, MOD, MUL,] + compare_lst + boolean_lst,
    POW: [LAMBDA, ADD, SUB, POS, NEG, MOD, MUL, DIV, POW] + compare_lst + boolean_lst,
    }

_is_name = re.compile(r'\A[a-zA-z_]\w*\Z').match

head_order = [NUMBER, SYMBOL, POS, ADD, SUB, MOD, MUL, DIV, POW,
              NEG,
              BOR, BXOR, BAND, INVERT, EQ, NE, LT, GT, LE, GE,
              OR, AND, NOT, LAMBDA, TUPLE
              ]
def tree_sort(a, b):
    h1,h2 = a.tree[0], b.tree[0]
    c = cmp(head_order.index(h1), head_order.index(h2))
    if c: return c
    t1,t2 = a.tree[1], b.tree[1]
    if h1 is SYMBOL or h1 is NUMBER:
        return cmp(t1, t2)
    c = cmp(len(t1), len(t2))
    if c: return c
    for i1,i2 in zip(t1,t2):
        c = tree_sort(i1, i2)
        if c: return c
    return 0

class PrimitiveAlgebra(BasicAlgebra):

    commutative_add = None
    commutative_mul = None

    def __new__(cls, tree, head=None):
        if head is None:
            return cls.convert(tree)
        if isinstance(tree, cls):
            return tree
        if type(tree) is not tuple:
            tree = (head, tree)
        obj = object.__new__(cls)
        obj.tree = tree
        return obj

    def __repr__(self):
        return str(self.tree)

    @classmethod
    def convert(cls, obj):
        if isinstance(obj, (str, unicode)):
            obj = string2PrimitiveAlgebra(obj)
        if hasattr(obj, 'as_primitive'):
            return obj.as_primitive()
        if isinstance(obj, cls):
            return obj
        return PrimitiveAlgebra(obj, head=SYMBOL)

    def as_primitive(self):
        return self

    def as_algebra(self, cls, source=None):
        head, rest = self.tree
        if head is NUMBER:
            return cls(rest, head=NUMBER)
        if head is SYMBOL:
            return cls(rest, head=SYMBOL)
        if head is ADD:
            return cls.Add([r.as_algebra(cls) for r in rest])
        if head is SUB:
            return rest[0].as_algebra(cls) - cls.Add([r.as_algebra(cls) for r in rest[1:]])
        if head is MUL:
            return cls.Mul([r.as_algebra(cls) for r in rest])
        if head is DIV:
            return rest[0].as_algebra(cls) / cls.Mul([r.as_algebra(cls) for r in rest[1:]])
        if head is POW:
            return cls.Pow(*[r.as_algebra(cls) for r in rest])
        if head is NEG:
            return -(rest[0].as_algebra(cls))
        if head is POS:
            return +(rest[0].as_algebra(cls))
        raise NotImplementedError('as_algebra(%s): %s' % (cls, self))

    def __str__(self):
        head, rest = self.tree
        if head is NUMBER or head is SYMBOL:
            return str(rest)
        if head is APPLY:
            func = rest[0]
            args = rest[1:]
            s = str(func)
            if _is_name(s):
                return '%s(%s)' % (s, ', '.join(map(str,args)))
            return '(%s)(%s)' % (s, ', '.join(map(str,args)))
        if head is LAMBDA:
            args = rest[0]
            body = rest[1]
            return 'lambda %s: %s' % (str(args)[1:-1], body)
        if head is TUPLE:
            return '(%s)' % (', '.join(map(str,rest)))
        if head is ADD:
            if self.commutative_add:
                rest = sorted(rest, cmp=tree_sort)
            r = ''
            for t in rest:
                h = t.tree[0]
                while h is POS:
                    t = t.tree[1][0]<commit_msg>Fix str output of nontrivial symbol/number content.<commit_after>
from __future__ import absolute_import
import types
import re
import compiler
from compiler import ast

from .algebraic_structures import BasicAlgebra

OR = intern(' or ')
AND = intern(' and ')
NOT = intern('not ')

LT = intern('<')
LE = intern('<=')
GT = intern('>')
GE = intern('>=')
EQ = intern('==')
NE = intern('!=')

BAND = intern('&')
BOR = intern('|')
BXOR = intern('^')
INVERT = intern('~')

POS = intern('+')
NEG = intern('-')
ADD = intern(' + ')
SUB = intern(' - ')
MOD = intern('%')
MUL = intern('*')
DIV = intern('/')
POW = intern('**')

NUMBER = intern('N')
SYMBOL = intern('S')
APPLY = intern('A')
TUPLE = intern('T')
LAMBDA = intern('L')

# XXX: Unimplemented expression parts:
# XXX: LeftShift, RightShift, List*, Subscript, Slice, KeyWord, GetAttr, Ellipsis
# XXX: function calls assume no optional nor *args nor *kwargs, same applies to lambda

boolean_lst = [AND, OR, NOT]
compare_lst = [LT, LE, GT, GE, EQ, NE]
bit_lst = [BAND, BOR, BXOR, INVERT]
arith_lst = [POS, NEG, ADD, SUB, MOD, MUL, DIV, POW]
parentheses_map = {
    OR: [LAMBDA],
    AND: [LAMBDA, OR],
    NOT: [LAMBDA, AND, OR],
    LT: [LAMBDA] + boolean_lst,
    LE: [LAMBDA] + boolean_lst,
    GT: [LAMBDA] + boolean_lst,
    GE: [LAMBDA] + boolean_lst,
    EQ: [LAMBDA] + boolean_lst,
    NE: [LAMBDA] + boolean_lst,
    BOR: [LAMBDA] + compare_lst + boolean_lst,
    BXOR: [LAMBDA, BOR] + compare_lst + boolean_lst,
    BAND: [LAMBDA, BOR, BXOR] + compare_lst + boolean_lst,
    INVERT: [LAMBDA, BOR, BXOR, BAND] + compare_lst + boolean_lst,
    ADD: [LAMBDA] + compare_lst + boolean_lst,
    SUB: [LAMBDA, ADD] + compare_lst + boolean_lst,
    POS: [LAMBDA, ADD, SUB] + compare_lst + boolean_lst,
    NEG: [LAMBDA, ADD, SUB] + compare_lst + boolean_lst,
    MOD: [LAMBDA, ADD, SUB, POS, NEG] + compare_lst + boolean_lst,
    MUL: [LAMBDA, ADD, SUB, POS, NEG, MOD] + compare_lst + boolean_lst,
    DIV: [LAMBDA, ADD, SUB, POS, NEG, MOD, MUL,] + compare_lst + boolean_lst,
    POW: [LAMBDA, ADD, SUB, POS, NEG, MOD, MUL, DIV, POW] + compare_lst + boolean_lst,
    }

_is_name = re.compile(r'\A[a-zA-z_]\w*\Z').match
_is_number = re.compile(r'\A[0-9.+-]+\Z').match

head_order = [NUMBER, SYMBOL, POS, ADD, SUB, MOD, MUL, DIV, POW,
              NEG,
              BOR, BXOR, BAND, INVERT, EQ, NE, LT, GT, LE, GE,
              OR, AND, NOT, LAMBDA, TUPLE
              ]
def tree_sort(a, b):
    h1,h2 = a.tree[0], b.tree[0]
    c = cmp(head_order.index(h1), head_order.index(h2))
    if c: return c
    t1,t2 = a.tree[1], b.tree[1]
    if h1 is SYMBOL or h1 is NUMBER:
        return cmp(t1, t2)
    c = cmp(len(t1), len(t2))
    if c: return c
    for i1,i2 in zip(t1,t2):
        c = tree_sort(i1, i2)
        if c: return c
    return 0

class PrimitiveAlgebra(BasicAlgebra):

    commutative_add = None
    commutative_mul = None

    def __new__(cls, tree, head=None):
        if head is None:
            return cls.convert(tree)
        if isinstance(tree, cls):
            return tree
        if type(tree) is not tuple:
            tree = (head, tree)
        obj = object.__new__(cls)
        obj.tree = tree
        return obj

    def __repr__(self):
        return str(self.tree)

    @classmethod
    def convert(cls, obj):
        if isinstance(obj, (str, unicode)):
            obj = string2PrimitiveAlgebra(obj)
        if hasattr(obj, 'as_primitive'):
            return obj.as_primitive()
        if isinstance(obj, cls):
            return obj
        return PrimitiveAlgebra(obj, head=SYMBOL)

    def as_primitive(self):
        return self

    def as_algebra(self, cls, source=None):
        head, rest = self.tree
        if head is NUMBER:
            return cls(rest, head=NUMBER)
        if head is SYMBOL:
            return cls(rest, head=SYMBOL)
        if head is ADD:
            return cls.Add([r.as_algebra(cls) for r in rest])
        if head is SUB:
            return rest[0].as_algebra(cls) - cls.Add([r.as_algebra(cls) for r in rest[1:]])
        if head is MUL:
            return cls.Mul([r.as_algebra(cls) for r in rest])
        if head is DIV:
            return rest[0].as_algebra(cls) / cls.Mul([r.as_algebra(cls) for r in rest[1:]])
        if head is POW:
            return cls.Pow(*[r.as_algebra(cls) for r in rest])
        if head is NEG:
            return -(rest[0].as_algebra(cls))
        if head is POS:
            return +(rest[0].as_algebra(cls))
        raise NotImplementedError('as_algebra(%s): %s' % (cls, self))

    def __str__(self):
        head, rest = self.tree
        if head is NUMBER:
            s = str(rest)
            if not _is_number(s):
                s = '((%s))' % (s)
            return s
        if head is SYMBOL:
            s = str(rest)
            if not _is_name(s):
                s = '((%s))' % (s)
            return s
        if head is APPLY:
            func = rest[0]
            args = rest[1:]
            s = str(func)
            if _is_name(s):
                return '%s(%s)' % (s, ', '.join(map(str,args)))
            return '(%s)(%s)' % (s, ', '.join(map(str,args)))
        if head is LAMBDA:
            args = rest[0]
            body = rest[1]
            return 'lambda %s: %s' % (str(args)[1:-1], body)
        if head is TUPLE:
            return '(%s)' % (', '.join(map(str,rest)))
        if head is ADD:
            if self.commutative_add:
                rest = sorted(rest, cmp=tree_sort)
            r = ''
            for t in rest:
                h = t.tree[0]
                while h is POS:
                    t = t.tree[1][0]
{'fraction_non_alphanumeric': 0.14566929133858267, 'fraction_numerical': 0.007324665812122322, 'mean_word_length': 2.247992863514719, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3078950', 'n_tokens_mistral': 4054, 'n_tokens_neox': 3951, 'n_words': 1113}