<commit_before>#!/usr/bin/python

from lib import *
import unittest

class TestLib(unittest.TestCase):
    def setUp(self):
        lines = """
        C17,2541,6525,2,END
        C13,END
        C17,2545,6523,2,END
        C17,2664,6506,2,END
        C17,2668,6508,2,END
        C17,2670,6512,2,END
        C17,2663,6534,2,END
        C17,2667,6534,2,END
        C17,2666,6540,2,END
        C17,2645,6543,2,END
        C17,2645,6537,2,END
        C17,2643,6531,2,END
        C17,2633,6527,2,END
        C17,2573,6535,2,END
        C17,2573,6538,2,END
        C14,END
        """
        self.instructions = []
        for line in lines.split("\n"):
            line = line.strip()
            if len(line):
                self.instructions.append(Instruction(line))

        self.glyph = Glyph(self.instructions)
   
    def test_reversing(self):
        text = "\n".join([i.line for i in self.glyph._reversed_instructions()])
        expected_text = """
        C17,2573,6538,2,END
        C13,END
        C17,2573,6535,2,END
        C17,2633,6527,2,END
        C17,2643,6531,2,END
        C17,2645,6537,2,END
        C17,2645,6543,2,END
        C17,2666,6540,2,END
        C17,2667,6534,2,END
        C17,2663,6534,2,END
        C17,2670,6512,2,END
        C17,2668,6508,2,END
        C17,2664,6506,2,END
        C17,2545,6523,2,END
        C17,2541,6525,2,END
        C14,END
        """
        expected_text = "\n".join([l.strip() for l in expected_text.strip().split("\n")])

        self.assertEqual(expected_text, text)

if __name__ == '__main__':
    unittest.main()<commit_msg>Add test for greedy ordering<commit_after>#!/usr/bin/python

from lib import *
import unittest

from itertools import tee, izip

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return izip(a, b)

class TestLib(unittest.TestCase):
    def makeGlyphFromText(self, lines):
        instructions = []
        for line in lines.split("\n"):
            line = line.strip()
            if len(line):
                instructions.append(Instruction(line))

        return Glyph(instructions)

    def setUp(self):
        lines = """
        C17,2541,6525,2,END
        C13,END
        C17,2545,6523,2,END
        C17,2664,6506,2,END
        C17,2668,6508,2,END
        C17,2670,6512,2,END
        C17,2663,6534,2,END
        C17,2667,6534,2,END
        C17,2666,6540,2,END
        C17,2645,6543,2,END
        C17,2645,6537,2,END
        C17,2643,6531,2,END
        C17,2633,6527,2,END
        C17,2573,6535,2,END
        C17,2573,6538,2,END
        C14,END
        """

        self.glyph = self.makeGlyphFromText(lines)

    def test_reversing(self):
        text = "\n".join([i.line for i in self.glyph._reversed_instructions()])
        expected_text = """
        C17,2573,6538,2,END
        C13,END
        C17,2573,6535,2,END
        C17,2633,6527,2,END
        C17,2643,6531,2,END
        C17,2645,6537,2,END
        C17,2645,6543,2,END
        C17,2666,6540,2,END
        C17,2667,6534,2,END
        C17,2663,6534,2,END
        C17,2670,6512,2,END
        C17,2668,6508,2,END
        C17,2664,6506,2,END
        C17,2545,6523,2,END
        C17,2541,6525,2,END
        C14,END
        """
        expected_text = "\n".join([l.strip() for l in expected_text.strip().split("\n")])

        self.assertEqual(expected_text, text)

    def test_greedy_sorting(self):
        """
        Exercise reorder_greedy() with arbitrary input (not chosen for any specific reason)
        """
        inputs = [self.glyph, 
            self.makeGlyphFromText("""
                C17,2638,6563,2,END
                C13,END
                C17,2679,6558,2,END
                C17,2677,6569,2,END
                C17,2663,6573,2,END
                C14,END
                """),
            self.makeGlyphFromText("""
                C17,2675,6524,2,END
                C13,END
                C17,2686,6491,2,END
                C17,2698,6485,2,END
                C17,2706,6495,2,END
                C17,2697,6502,2,END
                C17,2690,6509,2,END
                C17,2688,6510,2,END
                C17,2675,6524,2,END
                C14,END
                """),
            self.makeGlyphFromText("""
                C17,2644,6558,2,END
                C13,END
                C17,2647,6544,2,END
                C17,2665,6542,2,END
                C17,2661,6556,2,END
                C17,2660,6557,2,END
                C17,2644,6558,2,END
                C14,END
                """),
            self.makeGlyphFromText("""
                C17,2638,6563,2,END
                C13,END
                C17,2679,6558,2,END
                C17,2677,6569,2,END
                C17,2663,6573,2,END
                C14,END
                C17,2726,6599,2,END
                C13,END
                """),
            self.makeGlyphFromText("""
                C17,2675,6524,2,END
                C13,END
                C17,2686,6491,2,END
                C17,2698,6485,2,END
                C17,2706,6495,2,END
                C17,2697,6502,2,END
                C17,2690,6509,2,END
                C17,2688,6510,2,END
                C17,2675,6524,2,END
                C14,END
                """),
            ]

        ordered = reorder_greedy(inputs)


        print 'Inputs: {}'.format(map(lambda (g1, g2): g1.distance_to(g2), pairwise(inputs)))

        print 'ordered: {}'.format(map(lambda (g1, g2): g1.distance_to(g2), pairwise(ordered)))
        self.assertEqual(ordered[0], inputs[0])
        self.assertLessEqual(ordered[0].distance_to(ordered[1]), inputs[0].distance_to(ordered[2]))

if __name__ == '__main__':
    unittest.main()
{'fraction_non_alphanumeric': 0.13332162304584577, 'fraction_numerical': 0.17828912699806782, 'mean_word_length': 1.7140133460438514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6373294', 'n_tokens_mistral': 2659, 'n_tokens_neox': 2131, 'n_words': 262}