<commit_before># Will return the set of polynomial types it intersects with
#   (W_++, W+-, W^-1 ++, L_2, L_2^R etc)
# 1: W++
# 2: W+-
# 3: W-+
# 4: W--
# 5: Winv++
# 6: Winv+-
# 7: Winv-+
# 8: Winv--
# 9: L2
# 10: L2inv
mem = {}


def types(perm):
    interset = mem.get(perm)
    if interset is None:
        interset = set([])
        for i in range(len(perm)+1):
            part1 = perm[0:i]
            part2 = perm[i:]

            if is_incr(part1):
                if is_incr(part2):
                    interset.add(1)
                if is_decr(part2):
                    interset.add(2)

            if is_decr(part1):
                if is_incr(part2):
                    interset.add(3)
                if is_decr(part2):
                    interset.add(4)

        flipperm = perm.inverse()
        for i in range(len(perm)+1):
            part1 = flipperm[0:i]
            part2 = flipperm[i:]

            if is_incr(part1):
                if is_incr(part2):
                    interset.add(5)
                if is_decr(part2):
                    interset.add(6)

            if is_decr(part1):
                if is_incr(part2):
                    interset.add(7)
                if is_decr(part2):
                    interset.add(8)
        if in_L2(perm):
            interset.add(9)
        if in_L2(perm.reverse()):
            interset.add(10)
    return interset


def is_decr(L):
    for i in range(len(L) - 1):
        if L[i] < L[i+1]:
            return False
    return True


def is_incr(L):
    for i in range(len(L) - 1):
        if L[i] > L[i+1]:
            return False
    return True


def in_L2(L):
    n = len(L)
    if n == 0 or n == 1:
        return True
    if L[-1] == n-1:
        return in_L2(L[0:n-1])
    elif L[-1] == n-1 and L[-2] == n-1:
        return in_L2(L[0:n-2])
    else:
        return False


def is_polynomial(basis):
    overallinterset = set([])
    for perm in basis:
        overallinterset = overallinterset.union(types(perm))
        if len(overallinterset) == 10:
            return True
    return False


def is_non_polynomial(basis): return not is_polynomial(basis)<commit_msg>fix bug in poly check code<commit_after># Will return the set of polynomial types it intersects with
#   (W_++, W+-, W^-1 ++, L_2, L_2^R etc)
# 1: W++
# 2: W+-
# 3: W-+
# 4: W--
# 5: Winv++
# 6: Winv+-
# 7: Winv-+
# 8: Winv--
# 9: L2
# 10: L2inv
mem = {}


def types(perm):
    interset = mem.get(perm)
    if interset is None:
        interset = set([])
        for i in range(len(perm)+1):
            part1 = perm[0:i]
            part2 = perm[i:]

            if is_incr(part1):
                if is_incr(part2):
                    interset.add(1)
                if is_decr(part2):
                    interset.add(2)

            if is_decr(part1):
                if is_incr(part2):
                    interset.add(3)
                if is_decr(part2):
                    interset.add(4)

        flipperm = perm.inverse()
        for i in range(len(perm)+1):
            part1 = flipperm[0:i]
            part2 = flipperm[i:]

            if is_incr(part1):
                if is_incr(part2):
                    interset.add(5)
                if is_decr(part2):
                    interset.add(6)

            if is_decr(part1):
                if is_incr(part2):
                    interset.add(7)
                if is_decr(part2):
                    interset.add(8)
        if in_L2(perm):
            interset.add(9)
        if in_L2(perm.reverse()):
            interset.add(10)
    return interset


def is_decr(L):
    for i in range(len(L) - 1):
        if L[i] < L[i+1]:
            return False
    return True


def is_incr(L):
    for i in range(len(L) - 1):
        if L[i] > L[i+1]:
            return False
    return True


def in_L2(L):
    n = len(L)
    if n == 0 or n == 1:
        return True
    if L[-1] == n - 1:
        return in_L2(L[0:n - 1])
    elif L[-1] == n - 2 and L[-2] == n - 1:
        return in_L2(L[0:n - 2])
    else:
        return False


def is_polynomial(basis):
    overallinterset = set([])
    for perm in basis:
        overallinterset = overallinterset.union(types(perm))
        if len(overallinterset) == 10:
            return True
    return False


def is_non_polynomial(basis): return not is_polynomial(basis)
{'fraction_non_alphanumeric': 0.12912151256629006, 'fraction_numerical': 0.032280378141572516, 'mean_word_length': 1.4261744966442953, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3605141', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1542, 'n_words': 377}