<commit_before>from classes.semiring import *
from classes.graph import *
from classes.image import *
from classes.solver import *

from cProfile import Profile
from pstats import Stats


def prt_m(image):
    for i in range(image.get_size()[1]):
        for v in image.get_data()[i*image.get_size()[0]:(i+1)*image.get_size()[0]]:
            print '%03d'%v,
        print ''


def process_img(img, patterns, previous_vertex, offset=0, vertices=None, edges=None):

    if vertices is None:
        vertices = dict()
    if edges is None:
        edges = set()

    for p in patterns:

        if patterns[p].get_size()[0] > img.get_size()[0]:
            continue

        print 'Processing "%c"'%p
        img_left, img_right = img.split_vertical(patterns[p].get_size()[0])
        print 'LEFT'
        prt_m(img_left)
        print 'RIGHT'
        prt_m(img_right)

        current_vertex = None
        current_key = offset+patterns[p].get_size()[0]
        if vertices.has_key(current_key):
            for vertex in vertices[current_key]:
                if vertex.get_name() == p:
                    current_vertex = vertex
                    break
        if current_vertex is None:
            current_vertex = Vertex(p)

        if vertices.has_key(current_key):
            vertices[current_key].append(current_vertex)
        else:
            vertices[current_key] = [current_vertex]
        sigma = img_left.reduce(lambda accumulator, x, y: accumulator + (x - y)**2, 0, patterns[p])
        print (offset, offset+patterns[p].get_size()[0], p, sigma)

        edge = Edge(previous_vertex, current_vertex, (sigma, p))
        edges.add(edge)

        process_img(img_right, patterns, current_vertex, offset+patterns[p].get_size()[0], vertices, edges)

    return vertices, edges


def build_problem(main_img, patterns_imgs, Semiring=SemiringArgminPlusElement):

    image = MatrixPointer(list(main_img.getdata()), (main_img.size[0], main_img.size[1]))
    patterns = dict()
    prt_m(image)
    for p in patterns_imgs:
        patterns[p] = MatrixPointer(list(patterns_imgs[p].getdata()), (patterns_imgs[p].size[0], patterns_imgs[p].size[1]))
        prt_m(patterns[p])
        print ''

    start = Vertex('start')
    end = Vertex('end')
    fake_edge = Edge(start, end, Semiring.get_zero())

    min_width = min(patterns[p].get_size()[0] for p in patterns)
    if min_width > image.get_size()[0]:
        return Graph([start, end], fake_edge)

    profile = Profile()
    profile.enable()
    vertices, edges = process_img(image, patterns, start)
    profile.disable()
    Stats(profile).sort_stats('time').print_stats()
    vertices['start'] = [start]
    vertices['end'] = [end]
    if vertices.has_key(image.get_size()[0]):
        edges = edges.union(Edge(v, end, Semiring.get_unity())
                            for v in vertices[image.get_size()[0]])

    problem = DynamicProgramming(sum(vertices.values(), []), edges)
    problem.set_start(start)<commit_msg>Removed redundant prints in example<commit_after>from classes.semiring import *
from classes.graph import *
from classes.image import *
from classes.solver import *

from cProfile import Profile
from pstats import Stats


def process_img(img, patterns, previous_vertex, offset=0, vertices=None, edges=None):

    if vertices is None:
        vertices = dict()
    if edges is None:
        edges = set()

    for p in patterns:

        if patterns[p].get_size()[0] > img.get_size()[0]:
            continue

        img_left, img_right = img.split_vertical(patterns[p].get_size()[0])

        current_vertex = None
        current_key = offset+patterns[p].get_size()[0]
        if vertices.has_key(current_key):
            for vertex in vertices[current_key]:
                if vertex.get_name() == p:
                    current_vertex = vertex
                    break
        if current_vertex is None:
            current_vertex = Vertex(p)

        if vertices.has_key(current_key):
            vertices[current_key].append(current_vertex)
        else:
            vertices[current_key] = [current_vertex]
        sigma = img_left.reduce(lambda accumulator, x, y: accumulator + (x - y)**2, 0, patterns[p])
        print (offset, offset+patterns[p].get_size()[0], p, sigma)

        edge = Edge(previous_vertex, current_vertex, (sigma, p))
        edges.add(edge)

        process_img(img_right, patterns, current_vertex, offset+patterns[p].get_size()[0], vertices, edges)

    return vertices, edges


def build_problem(main_img, patterns_imgs, Semiring=SemiringArgminPlusElement):

    image = MatrixPointer(list(main_img.getdata()), (main_img.size[0], main_img.size[1]))
    patterns = dict()
    for p in patterns_imgs:
        patterns[p] = MatrixPointer(list(patterns_imgs[p].getdata()), (patterns_imgs[p].size[0], patterns_imgs[p].size[1]))

    start = Vertex('start')
    end = Vertex('end')
    fake_edge = Edge(start, end, Semiring.get_zero())

    min_width = min(patterns[p].get_size()[0] for p in patterns)
    if min_width > image.get_size()[0]:
        return Graph([start, end], fake_edge)

    profile = Profile()
    profile.enable()
    vertices, edges = process_img(image, patterns, start)
    profile.disable()
    Stats(profile).sort_stats('time').print_stats()
    vertices['start'] = [start]
    vertices['end'] = [end]
    if vertices.has_key(image.get_size()[0]):
        edges = edges.union(Edge(v, end, Semiring.get_unity())
                            for v in vertices[image.get_size()[0]])

    problem = DynamicProgramming(sum(vertices.values(), []), edges)
    problem.set_start(start)
{'fraction_non_alphanumeric': 0.1297451434681875, 'fraction_numerical': 0.007128854036713598, 'mean_word_length': 2.9801418439716314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5561674', 'n_tokens_mistral': 1781, 'n_tokens_neox': 1737, 'n_words': 378}