<commit_before>
from .permutation_sets import PointPermutationSet, InputPermutationSet, SimpleGeneratingRule, GeneratingRule, OverlayGeneratingRule, StaticPermutationSet, UniversePermutationSet, EmptyPermutationSet
from permuta import Permutation, Permutations
from permuta.misc import binary_search
from copy import deepcopy
from .misc.cache import Cache
import marshal

# def generate_small_input(perm_prop):
#     bound = 2
#     ocreated = {}
#     for l in range(bound+1):
#         ocreated.setdefault(l, [])
#         for perm in Permutations(l):
#             if perm_prop(perm):
#                 ocreated[l].append(perm)
#     return ocreated

def generate_all_of_length(max_n, S, inp, min_n=0):

    inp = deepcopy(inp)

    for n in range(min_n,max_n+1):
        inp.setdefault(n, [])
        for perm in S.generate_of_length(n, inp):
            inp[n].append(perm)

    return inp

def find_allowed_neighbors_classical_perm_prop(settings):

    neighb = {}
    for i in range(len(settings.sets)):
        for j in range(len(settings.sets)):
            for di in range(-1, 2):
                for dj in range(-1, 2):
                    if di == 0 and dj == 0:
                        continue

                    neighb.setdefault((settings.sets[i], di, dj), set())

                    a = min(0, di)
                    b = min(0, dj)
                    G = GeneratingRule({ (-a, -b): settings.sets[i], (di-a, dj-b): settings.sets[j] })

                    ok = True
                    for l in range(settings.perm_bound+1):
                        for perm in G.generate_of_length(l, settings.sinput.permutations):
                            perm = Permutation(list(perm))
                            if not settings.sinput.contains(perm):
                                ok = False
                                break
                        if not ok:
                            break
                    if ok:
                        neighb[(settings.sets[i], di, dj)].add(settings.sets[j])
    return neighb


def find_allowed_neighbors(settings):
    small_bound = 2
    small_perms = [[] for i in range(len(settings.sets))]
    for l in range(small_bound+1):
        for i in range(len(settings.sets)):
            if settings.sets[i]:
                for perm in settings.sets[i].generate_of_length(l, settings.sinput.permutations):
                    small_perms[i].append(perm)
    for i in range(len(settings.sets)):
        small_perms[i] = sorted(small_perms[i])

    key = small_perms
    if not settings.ignore_cache and Cache.contains(key):
        neighb = Cache.get(key)
    else:
        neighb = {}

        for i in range(len(settings.sets)):
            for j in range(len(settings.sets)):
                for di in range(-1, 2):
                    for dj in range(-1, 2):
                        if di == 0 and dj == 0:
                            continue

                        # neighb.setdefault((cur_elem, di, dj), set())
                        neighb.setdefault((i, di, dj), set())

                        a = min(0, di)
                        b = min(0, dj)
                        # G = GeneratingRule({ (-a, -b): cur_elem, (di-a, dj-b): other_elem })
                        G = GeneratingRule({ (-a, -b): settings.sets[i], (di-a, dj-b): settings.sets[j] })

                        ok = True
                        for l in range(settings.perm_bound+1):
                            perms = list(G.generate_of_length(l, settings.sinput.permutations))
                            if len(perms) != len(set(perms)):
                                ok = False
                                break
                        if ok:
                            # neighb[(cur_elem, di, dj)].add(other_elem)
                            neighb[(i, di, dj)].add(j)

        if not settings.ignore_cache:
            Cache.put(key,neighb)

    res = {}
    for (i,di,dj),v in neighb.items():
        res[(settings.sets[i],di,dj)] = set([ settings.sets[j] for j in v ])
    return res


# def generate_rules(settings, n, m,
#         allowed_neighbors=None,
#         use_allowed_neighbors=True,
#         allowed_neighbors_cpp=None,
#     ):
#
#     if allowed_neighbors is None and use_allowed_neighbors:
#         allowed_neighbors = find_allowed_neighbors(settings)
#     if not use_allowed_neighbors:
#         allowed_neighbors = None
#
#     if settings.sinput.is_classical and allowed_neighbors_cpp is None:
#         allowed_neighbors_cpp = find_allowed_neighbors_classical_perm_prop(settings)
#
#     # key = (n,m,perm_bound, sets, cnt, allowed_neighbors)
#     # print('a')
#     # if Cache.contains(key):
#     #     res = Cache.get(key)
#     #     print('b cache')
#     #     return res
#
#     def valid(rule):
#         if n == 1 and m == 1:
#             # edge case: all 1x1 rules are valid
#             if rule[0][0] and not rule[0][0].can_be_alone():
#                 return False
#             return True
#
#         if all( x is None for x in rule[0] ):
#             return False
#
#         for y in range(m):
#             found = False
#             for x in range(n):
#                 if rule[x][y] is not None:
#                     found = True
#                     break
#
#             if not found:
#                 return False
#
#         return True
#
#     def signum(n):
#         return 1 if n > 0 else -1 if n < 0 else 0
#
#     ssets = set(settings.sets)
#     def gen(i, j):
#         if j == m:
#             for rule in gen(i + 1, 0):
#                 if i + 1 == n or any( x is not None for x in rule[i + 1] ):
#                     yield rule
#         elif i == n:
#             yield [ [ None for y in range(m) ] for x in range(n) ]
#         else:
#             for trule in gen(i, j + 1):
#                 rule = [ [ trule[x][y] for y in range(m) ] for x in range(n) ]
#                 left = settings.max_non_empty - sum( rule[x][y] is not None for x in range(n) for y in range(m) )
#                 at_most = settings.mn_at_most - sum( 0 if rule[x][y] is None else rule[x][y].min_length(settings.sinput.permutations) for x in range(n) for y in range(m) )
#
#                 ss = set(ssets)
#                 if use_allowed_neighbors:
#                     for di in range(-1, 2):
#                         for dj in range(-1, 2):
#                             ci, cj = i+di, j+dj
#                             if (ci, cj) > (i, j) and 0 <= ci < n and 0 <= cj < m:
#                                 curs = allowed_neighbors[(rule[ci][cj], -di, -dj)]
#                                 ss &= curs
#
#                 if allowed_neighbors_cpp is not None:
#                     for ci in range(i,n):
#                         for cj in range(m):
#                             if (ci, cj) > (i, j):
#                                 di = signum(ci-i)
#                                 dj = signum(cj-j)
#                                 curs = allowed_neighbors_cpp[(rule[ci][cj], -di, -dj)]
#                                 ss &= curs
#
#                 for s in ss:
#                     if s is not None:
#                         if left == 0 or s.min_length(settings.sinput.permutations) > at_most:
#                             continue
#                     rule[i][j] = s
#                     yield rule
#
#     res = []
#     a, b = 0, 0
#     for rule in gen(0, 0):
#         # if (n,m) == (3,3):
#         #     print(GeneratingRule(rule))
#         a += 1
#         if valid(rule):
#             b += 1
#             res.append(GeneratingRule(rule))
#
#     # print('b')
#     # Cache.put(key,res)
#     # print('c')
#     return res




def generate_rules(settings, n, m,
        allowed_neighbors=None,
        use_allowed_neighbors=True,
        allowed_neighbors_cpp=None,
    ):

    if allowed_neighbors is None and use_allowed_neighbors:
        allowed_neighbors = find_allowed_neighbors(settings)
    if not use_allowed_neighbors:
        allowed_neighbors = None

    if settings.sinput.is_classical and allowed_neighbors_cpp is None:
        allowed_neighbors_cpp = find_allowed_neighbors_classical_perm_prop(settings)

    # key = (n,m,perm_bound, sets, cnt, allowed_neighbors)
    # print('a')
    # if Cache.contains(key):
    #     res = Cache.get(key)
    #     print('b cache')
    #     return res

    def valid(rule):
        if n == 1 and m == 1:
            # edge case: all 1x1 rules are valid
            if rule[0][0] and not rule[0][0].can_be_alone():
                return False
            return True

        if all( x is None for x in rule[0] ):
            return False

        for y in range(m):
            found = False
            for x in range(n):
                if rule[x][y] is not None:
                    found = True
                    break

            if not found:
                return False

        return True

    def generates_subset(rule):
        for l in range(settings.perm_bound+1):
            found = set()
            for p in rule.generate_of_length(l, settings.sinput.permutations):
                if p in found or not settings.sinput.contains(p):
                    return False
                found.add(p)
        return True

    def signum(n):
        return 1 if n > 0 else -1 if n < 0 else 0

    ssets = set(settings.sets)
    def gen(i, j):
        if j == m:
            for rule in gen(i + 1, 0):
                if i + 1 == n or any( x is not None for x in rule[i + 1] ):
                    yield rule
        elif i == n:
            yield [ [ None for y in range(m) ] for x in range(n) ]
        else:
            for trule in gen(i, j + 1):
                rule = [ [ trule[x][y] for y in range(m) ] for x in range(n) ]
                left = settings.max_non_empty - sum( rule[x][y] is not None for x in range(n) for y in range(m) )
                at_most = settings.mn_at_most - sum( 0 if rule[x][y] is None else rule[x][y].min_length(settings.sinput.permutations) for x in range(n) for y in range(m) )

                ss = set(ssets)
                if use_allowed_neighbors:
                    for di in range(-1, 2):
                        for dj in range(-1, 2):
                            ci, cj = i+di, j+dj
                            if (ci, cj) > (i, j) and 0 <= ci < n and 0 <= cj < m:
                                curs = allowed_neighbors[(rule[ci][cj], -di, -dj)]
                                ss &= curs

                if allowed_neighbors_cpp is not None:
                    for ci in range(i,n):
                        for cj in range(m):
                            if (ci, cj) > (i, j):
                                di = signum(ci-i)
                                dj = signum(cj-j)
                                curs = allowed_neighbors_cpp[(rule[ci][cj], -di, -dj)]
                                ss &= curs

                for s in ss:
                    if s is not None:
                        if left == 0 or s.min_length(settings.sinput.permutations) > at_most:
                            continue
                    rule[i][j] = s
                    if generates_subset(GeneratingRule(rule)):
                        yield rule

    res = []
    a, b = 0, 0
    for rule in gen(0, 0):
        # if (n,m) == (3,3):
        #     print(GeneratingRule(rule))
        a += 1
        if valid(rule):
            b += 1
            res.append(GeneratingRule(rule))

    # print('b')
    # Cache.put(key,res)
    # print('c')<commit_msg>Disable intelligent generation, for now<commit_after>
from .permutation_sets import PointPermutationSet, InputPermutationSet, SimpleGeneratingRule, GeneratingRule, OverlayGeneratingRule, StaticPermutationSet, UniversePermutationSet, EmptyPermutationSet
from permuta import Permutation, Permutations
from permuta.misc import binary_search
from copy import deepcopy
from .misc.cache import Cache
import marshal

# def generate_small_input(perm_prop):
#     bound = 2
#     ocreated = {}
#     for l in range(bound+1):
#         ocreated.setdefault(l, [])
#         for perm in Permutations(l):
#             if perm_prop(perm):
#                 ocreated[l].append(perm)
#     return ocreated

def generate_all_of_length(max_n, S, inp, min_n=0):

    inp = deepcopy(inp)

    for n in range(min_n,max_n+1):
        inp.setdefault(n, [])
        for perm in S.generate_of_length(n, inp):
            inp[n].append(perm)

    return inp

def find_allowed_neighbors_classical_perm_prop(settings):

    neighb = {}
    for i in range(len(settings.sets)):
        for j in range(len(settings.sets)):
            for di in range(-1, 2):
                for dj in range(-1, 2):
                    if di == 0 and dj == 0:
                        continue

                    neighb.setdefault((settings.sets[i], di, dj), set())

                    a = min(0, di)
                    b = min(0, dj)
                    G = GeneratingRule({ (-a, -b): settings.sets[i], (di-a, dj-b): settings.sets[j] })

                    ok = True
                    for l in range(settings.perm_bound+1):
                        for perm in G.generate_of_length(l, settings.sinput.permutations):
                            perm = Permutation(list(perm))
                            if not settings.sinput.contains(perm):
                                ok = False
                                break
                        if not ok:
                            break
                    if ok:
                        neighb[(settings.sets[i], di, dj)].add(settings.sets[j])
    return neighb


def find_allowed_neighbors(settings):
    small_bound = 2
    small_perms = [[] for i in range(len(settings.sets))]
    for l in range(small_bound+1):
        for i in range(len(settings.sets)):
            if settings.sets[i]:
                for perm in settings.sets[i].generate_of_length(l, settings.sinput.permutations):
                    small_perms[i].append(perm)
    for i in range(len(settings.sets)):
        small_perms[i] = sorted(small_perms[i])

    key = small_perms
    if not settings.ignore_cache and Cache.contains(key):
        neighb = Cache.get(key)
    else:
        neighb = {}

        for i in range(len(settings.sets)):
            for j in range(len(settings.sets)):
                for di in range(-1, 2):
                    for dj in range(-1, 2):
                        if di == 0 and dj == 0:
                            continue

                        # neighb.setdefault((cur_elem, di, dj), set())
                        neighb.setdefault((i, di, dj), set())

                        a = min(0, di)
                        b = min(0, dj)
                        # G = GeneratingRule({ (-a, -b): cur_elem, (di-a, dj-b): other_elem })
                        G = GeneratingRule({ (-a, -b): settings.sets[i], (di-a, dj-b): settings.sets[j] })

                        ok = True
                        for l in range(settings.perm_bound+1):
                            perms = list(G.generate_of_length(l, settings.sinput.permutations))
                            if len(perms) != len(set(perms)):
                                ok = False
                                break
                        if ok:
                            # neighb[(cur_elem, di, dj)].add(other_elem)
                            neighb[(i, di, dj)].add(j)

        if not settings.ignore_cache:
            Cache.put(key,neighb)

    res = {}
    for (i,di,dj),v in neighb.items():
        res[(settings.sets[i],di,dj)] = set([ settings.sets[j] for j in v ])
    return res


# def generate_rules(settings, n, m,
#         allowed_neighbors=None,
#         use_allowed_neighbors=True,
#         allowed_neighbors_cpp=None,
#     ):
#
#     if allowed_neighbors is None and use_allowed_neighbors:
#         allowed_neighbors = find_allowed_neighbors(settings)
#     if not use_allowed_neighbors:
#         allowed_neighbors = None
#
#     if settings.sinput.is_classical and allowed_neighbors_cpp is None:
#         allowed_neighbors_cpp = find_allowed_neighbors_classical_perm_prop(settings)
#
#     # key = (n,m,perm_bound, sets, cnt, allowed_neighbors)
#     # print('a')
#     # if Cache.contains(key):
#     #     res = Cache.get(key)
#     #     print('b cache')
#     #     return res
#
#     def valid(rule):
#         if n == 1 and m == 1:
#             # edge case: all 1x1 rules are valid
#             if rule[0][0] and not rule[0][0].can_be_alone():
#                 return False
#             return True
#
#         if all( x is None for x in rule[0] ):
#             return False
#
#         for y in range(m):
#             found = False
#             for x in range(n):
#                 if rule[x][y] is not None:
#                     found = True
#                     break
#
#             if not found:
#                 return False
#
#         return True
#
#     def signum(n):
#         return 1 if n > 0 else -1 if n < 0 else 0
#
#     ssets = set(settings.sets)
#     def gen(i, j):
#         if j == m:
#             for rule in gen(i + 1, 0):
#                 if i + 1 == n or any( x is not None for x in rule[i + 1] ):
#                     yield rule
#         elif i == n:
#             yield [ [ None for y in range(m) ] for x in range(n) ]
#         else:
#             for trule in gen(i, j + 1):
#                 rule = [ [ trule[x][y] for y in range(m) ] for x in range(n) ]
#                 left = settings.max_non_empty - sum( rule[x][y] is not None for x in range(n) for y in range(m) )
#                 at_most = settings.mn_at_most - sum( 0 if rule[x][y] is None else rule[x][y].min_length(settings.sinput.permutations) for x in range(n) for y in range(m) )
#
#                 ss = set(ssets)
#                 if use_allowed_neighbors:
#                     for di in range(-1, 2):
#                         for dj in range(-1, 2):
#                             ci, cj = i+di, j+dj
#                             if (ci, cj) > (i, j) and 0 <= ci < n and 0 <= cj < m:
#                                 curs = allowed_neighbors[(rule[ci][cj], -di, -dj)]
#                                 ss &= curs
#
#                 if allowed_neighbors_cpp is not None:
#                     for ci in range(i,n):
#                         for cj in range(m):
#                             if (ci, cj) > (i, j):
#                                 di = signum(ci-i)
#                                 dj = signum(cj-j)
#                                 curs = allowed_neighbors_cpp[(rule[ci][cj], -di, -dj)]
#                                 ss &= curs
#
#                 for s in ss:
#                     if s is not None:
#                         if left == 0 or s.min_length(settings.sinput.permutations) > at_most:
#                             continue
#                     rule[i][j] = s
#                     yield rule
#
#     res = []
#     a, b = 0, 0
#     for rule in gen(0, 0):
#         # if (n,m) == (3,3):
#         #     print(GeneratingRule(rule))
#         a += 1
#         if valid(rule):
#             b += 1
#             res.append(GeneratingRule(rule))
#
#     # print('b')
#     # Cache.put(key,res)
#     # print('c')
#     return res




def generate_rules(settings, n, m,
        allowed_neighbors=None,
        use_allowed_neighbors=True,
        allowed_neighbors_cpp=None,
    ):

    if allowed_neighbors is None and use_allowed_neighbors:
        allowed_neighbors = find_allowed_neighbors(settings)
    if not use_allowed_neighbors:
        allowed_neighbors = None

    if settings.sinput.is_classical and allowed_neighbors_cpp is None:
        allowed_neighbors_cpp = find_allowed_neighbors_classical_perm_prop(settings)

    # key = (n,m,perm_bound, sets, cnt, allowed_neighbors)
    # print('a')
    # if Cache.contains(key):
    #     res = Cache.get(key)
    #     print('b cache')
    #     return res

    def valid(rule):
        if n == 1 and m == 1:
            # edge case: all 1x1 rules are valid
            if rule[0][0] and not rule[0][0].can_be_alone():
                return False
            return True

        if all( x is None for x in rule[0] ):
            return False

        for y in range(m):
            found = False
            for x in range(n):
                if rule[x][y] is not None:
                    found = True
                    break

            if not found:
                return False

        return True

    def generates_subset(rule):
        for l in range(settings.perm_bound+1):
            found = set()
            for p in rule.generate_of_length(l, settings.sinput.permutations):
                if p in found or not settings.sinput.contains(p):
                    return False
                found.add(p)
        return True

    def signum(n):
        return 1 if n > 0 else -1 if n < 0 else 0

    ssets = set(settings.sets)
    def gen(i, j):
        if j == m:
            for rule in gen(i + 1, 0):
                if i + 1 == n or any( x is not None for x in rule[i + 1] ):
                    yield rule
        elif i == n:
            yield [ [ None for y in range(m) ] for x in range(n) ]
        else:
            for trule in gen(i, j + 1):
                rule = [ [ trule[x][y] for y in range(m) ] for x in range(n) ]
                left = settings.max_non_empty - sum( rule[x][y] is not None for x in range(n) for y in range(m) )
                at_most = settings.mn_at_most - sum( 0 if rule[x][y] is None else rule[x][y].min_length(settings.sinput.permutations) for x in range(n) for y in range(m) )

                ss = set(ssets)
                if use_allowed_neighbors:
                    for di in range(-1, 2):
                        for dj in range(-1, 2):
                            ci, cj = i+di, j+dj
                            if (ci, cj) > (i, j) and 0 <= ci < n and 0 <= cj < m:
                                curs = allowed_neighbors[(rule[ci][cj], -di, -dj)]
                                ss &= curs

                if allowed_neighbors_cpp is not None:
                    for ci in range(i,n):
                        for cj in range(m):
                            if (ci, cj) > (i, j):
                                di = signum(ci-i)
                                dj = signum(cj-j)
                                curs = allowed_neighbors_cpp[(rule[ci][cj], -di, -dj)]
                                ss &= curs

                for s in ss:
                    if s is not None:
                        if left == 0 or s.min_length(settings.sinput.permutations) > at_most:
                            continue
                    rule[i][j] = s
                    # if generates_subset(GeneratingRule(rule)):
                    yield rule

    res = []
    a, b = 0, 0
    for rule in gen(0, 0):
        # if (n,m) == (3,3):
        #     print(GeneratingRule(rule))
        a += 1
        if valid(rule):
            b += 1
            res.append(GeneratingRule(rule))

    # print('b')
    # Cache.put(key,res)
    # print('c')
{'fraction_non_alphanumeric': 0.11070559610705596, 'fraction_numerical': 0.008255126868265555, 'mean_word_length': 1.332016210739615, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476607', 'n_tokens_mistral': 7323, 'n_tokens_neox': 6842, 'n_words': 1905}