<commit_before>from .. import combinators as c
from .. import maybeerror as me
import unittest as u


m = me.MaybeError

l = c.ConsList

def good(rest, state, result):
    return m.pure({'rest': rest, 'state': state, 'result': result})

iz1 = c.basic
(item1, lit1, sat1, not11, str1, oneOf1) = iz1.item, iz1.literal, iz1.satisfy, iz1.not1, iz1.string, iz1.oneOf
iz2 = c.position
(item2, lit2, sat2, not12, str2, oneOf2) = iz2.item, iz2.literal, iz2.satisfy, iz2.not1, iz2.string, iz2.oneOf
iz3 = c.count
(item3, lit3, sat3, not13, str3, oneOf3) = iz3.item, iz3.literal, iz3.satisfy, iz3.not1, iz3.string, iz3.oneOf



class BasicTokens(u.TestCase):

    def testItemBasic(self):
        self.assertEqual(m.zero, item1.parse(l(''), None))
        self.assertEqual(good(l('bcdef'), None, 'a'), item1.parse(l('abcdef'), None))

    def testLiteral(self):
        val = lit1(3)
        self.assertEqual(val.parse(l([3,4,5]), {}), good(l([4,5]), {}, 3))
        self.assertEqual(val.parse(l([4,5]), {}), m.zero)
    
    def testSatisfy(self):
        v1 = sat1(lambda x: x > 3).parse(l([1,2,3]), 'bye')
        self.assertEqual(m.zero, v1)
        v2 = sat1(lambda x: x < 3).parse(l([1,2,3]), 'hi')
        self.assertEqual(good(l([2,3]), 'hi', 1), v2)
    
    def testString(self):
        parser = str1('abc')
        v1 = parser.parse(l('abcdef'), None)
        self.assertEqual(good(l('def'), None, 'abc'), v1)
        v2 = parser.parse(l('abdef'), None)
        self.assertEqual(m.zero, v2)
    
    def testNot1(self):
        val = not11(lit1(2))
        self.assertEqual(val.parse(l([2,3,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,4,5]), {}), good(l([4,5]), {}, 3))
    
    def testOneOf(self):
        p = oneOf1('abc')
        self.assertEqual(p.parse(l('cqrs'), None), good(l('qrs'), None, 'c'))
        self.assertEqual(p.parse(l('aqrs'), None), good(l('qrs'), None, 'a'))
        self.assertEqual(p.parse(l('dqrs'), None), m.zero)



class PositionTokens(u.TestCase):

    def testItemPosition(self):
        self.assertEqual(m.zero, item2.parse(l(''), (1,1)))
        self.assertEqual(good(l('bcdef'), (1,2), 'a'), item2.parse(l('abcdef'), (1,1)))
        self.assertEqual(good(l('bcdef'), (2,1), '\n'), item2.parse(l('\nbcdef'), (1,1)))

    def testLiteral(self):
        val = lit2('3')
        self.assertEqual(val.parse(l('345'), (3,8)), good(l('45'), (3,9), '3'))
        self.assertEqual(val.parse(l('45'), (3,8)), m.zero)
    
    def testSatisfy(self):
        v1 = sat2(lambda x: int(x) > 3).parse(l('123'), (2,2))
        self.assertEqual(m.zero, v1)
        v2 = sat2(lambda x: int(x) < 3).parse(l('123'), (2,2))
        self.assertEqual(good(l('23'), (2,3), '1'), v2)
    
    def testString(self):
        parser = str2('abc')
        v1 = parser.parse(l('abcdef'), (4,3))
        self.assertEqual(good(l('def'), (4,6), 'abc'), v1)
        v2 = parser.parse(l('abdef'), (4,3))
        self.assertEqual(m.zero, v2)
    
    def testNot1(self):
        val = not12(lit2('2'))
        self.assertEqual(val.parse(l('234'), (1,1)), m.zero)
        self.assertEqual(val.parse(l('345'), (1,1)), good(l('45'), (1,2), '3'))
    
    def testOneOf(self):
        p = oneOf2('abc')
        self.assertEqual(p.parse(l('cqrs'), (3,4)), good(l('qrs'), (3,5), 'c'))
        self.assertEqual(p.parse(l('aqrs'), (8,1)), good(l('qrs'), (8,2), 'a'))
        self.assertEqual(p.parse(l('dqrs'), (2,2)), m.zero)



class CountTokens(u.TestCase):

    def testItemPosition(self):
        self.assertEqual(m.zero, item3.parse(l(''), 8))
        self.assertEqual(good(l('bcdef'), 6, 'a'), item3.parse(l('abcdef'), 5))
        self.assertEqual(good(l('bcdef'), 101, '\n'), item3.parse(l('\nbcdef'), 100))

    def testLiteral(self):
        val = lit3('3')
        self.assertEqual(val.parse(l('345'), 8), good(l('45'), 9, '3'))
        self.assertEqual(val.parse(l('45'), 8), m.zero)
    
    def testSatisfy(self):
        v1 = sat3(lambda x: int(x) > 3).parse(l('123'), 22)
        self.assertEqual(m.zero, v1)
        v2 = sat3(lambda x: int(x) < 3).parse(l('123'), 22)
        self.assertEqual(good(l('23'), 23, '1'), v2)
    
    def testString(self):
        parser = str3('abc')
        v1 = parser.parse(l('abcdef'), 43)
        self.assertEqual(good(l('def'), 46, 'abc'), v1)
        v2 = parser.parse(l('abdef'), 43)
        self.assertEqual(m.zero, v2)
    
    def testNot1(self):
        val = not13(lit3('2'))
        self.assertEqual(val.parse(l('234'), 61), m.zero)
        self.assertEqual(val.parse(l('345'), 61), good(l('45'), 62, '3'))
    
    def testOneOf(self):
        p = oneOf3('abc')
        self.assertEqual(p.parse(l('cqrs'), 4), good(l('qrs'), 5, 'c'))
        self.assertEqual(p.parse(l('aqrs'), 8), good(l('qrs'), 9, 'a'))
        self.assertEqual(p.parse(l('dqrs'), 7), m.zero)



class TestParser(u.TestCase):
    
    def testFmap(self):
        f = lambda x: x + 7
        v1 = c.fmap(f, c.pure(3)).parse('ab', 81)
        self.assertEqual(good('ab', 81, 10), v1)
        v2 = c.fmap(f, c.zero).parse('ab', 81)
        self.assertEqual(m.zero, v2)
        v3 = c.fmap(f, c.error('oops')).parse('ab', 81)
        self.assertEqual(m.error('oops'), v3)
        
    def testPure(self):
        val = c.pure(3).parse('abc', 2)
        self.assertEqual(good('abc', 2, 3), val)
    
    def testBind(self):
        two = c.bind(item1, lambda x: lit1(x))
        self.assertEqual(two.parse(l('abcde'), {}), m.zero)
        self.assertEqual(two.parse(l('aabcde'), {}), good(l('bcde'), {}, 'a'))

    def testAltBinaryRules(self):
        g1, g2, b, e, e2 = c.pure(3), c.pure('hi'), c.zero, c.error('oops'), c.error('2nd')
        r1, r3, r4 = good('abc', None, 3), m.zero, m.error('oops')
        self.assertEqual(c.alt(g1, g2).parse('abc', None), r1)
        self.assertEqual(c.alt(g1, b).parse('abc', None), r1)
        self.assertEqual(c.alt(g1, e).parse('abc', None), r1)
        self.assertEqual(c.alt(b, g1).parse('abc', None), r1)
        self.assertEqual(c.alt(b, b).parse('abc', None), r3)
        self.assertEqual(c.alt(b, e).parse('abc', None), r4)
        self.assertEqual(c.alt(e, g1).parse('abc', None), r4)
        self.assertEqual(c.alt(e, b).parse('abc', None), r4)
        self.assertEqual(c.alt(e, e2).parse('abc', None), r4)
    
    def testAltCornerCases(self):
        self.assertEqual(m.zero, 
                         c.alt().parse(l([1,2,3]), None))
        self.assertEqual(good(l([1,2,3]), None, 'h'), 
                         c.alt(c.pure('h')).parse(l([1,2,3]), None))
        self.assertEqual(m.error('oops'), 
                         c.alt(c.error('oops')).parse(l([1,2,3]), None))
        self.assertEqual(m.zero, 
                         c.alt(c.zero).parse(l([1,2,3]), None))
        p1 = c.alt(c.zero, lit1(1), lit1(2), c.error('d'))
        self.assertEqual(good(l([3,4]), None, 1), p1.parse(l([1,3,4]), None))
        self.assertEqual(good(l([3,4]), None, 2), p1.parse(l([2,3,4]), None))
        self.assertEqual(m.error('d'), p1.parse(l([3,3,4]), None))
    
    def testError(self):
        v1 = c.error('uh-oh').parse('abc', 123)
        self.assertEqual(m.error('uh-oh'), v1)
    
    def testCatchError(self):
        f1 = lambda e: c.pure(3)
        f2 = lambda e: c.error('dead again')
        # error -> good -- resumes parsing with tokens and state from before the error occurred
        self.assertEqual(good('123', [2,4], 3), c.catchError(f1, c.error('dead 1')).parse('123', [2, 4]))
        # good -> good (unaffected by this combinator)
        self.assertEqual(good('123', [2,4], 18), c.catchError(f1, c.pure(18)).parse('123', [2,4]))
        # error -> error
        self.assertEqual(m.error('dead again'), c.catchError(f2, c.error('dead 1')).parse('123', [2,4]))
        # good -> error is not possible with this combinator
        
    def testMapError(self):
        f = len
        v1 = c.mapError(f, c.error('abcdef')).parse('123abc', None)
        self.assertEqual(m.error(6), v1)
        v2 = c.mapError(f, c.zero).parse('123abc', None)
        self.assertEqual(m.zero, v2)
        v3 = c.mapError(f, c.pure(82)).parse('123abc', None)
        self.assertEqual(good('123abc', None, 82), v3)        

    def testPut(self):
        val = c.put('xyz')
        self.assertEqual(val.parse('abc', []), good('xyz', [], None))
    
    def testPutState(self):
        v1 = c.putState(29).parse('abc123', 2)
        self.assertEqual(good('abc123', 29, None), v1)
    
    def testUpdateState(self):
        v1 = c.updateState(lambda x: x * 4).parse('abc', 18)
        self.assertEqual(good('abc', 72, None), v1)
        
    def testCheck(self):
        val = c.check(lambda x: len(x) > 3, c.get)
        self.assertEqual(val.parse('abcde', []), good('abcde', [], 'abcde'))
        self.assertEqual(val.parse('abc', []), m.zero)
    
    def testMany0(self):
        val = c.many0(lit1(3))
        self.assertEqual(val.parse(l([4,4,4]), {}), good(l([4,4,4]), {}, []))
        self.assertEqual(val.parse(l([3,3,4,5]), {}), good(l([4,5]), {}, [3,3]))
    
    def testMany1(self):
        val = c.many1(lit1(3))
        self.assertEqual(val.parse(l([4,4,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,3,4,5]), {}), good(l([4,5]), {}, [3,3]))
    
    def testSeq(self):
        val = c.seq(item1, lit1(2), lit1(8))
        self.assertEqual(val.parse(l([3,2,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,2,8,16]), {}), good(l([16]), {}, [3,2,8]))
    
    def testApp(self):
        parser = c.app(lambda x,y,z: x + y * z, item1, sat1(lambda x: x > 2), item1)
        v1 = parser.parse(l([1,2,3,4,5]), 'hi')
        self.assertEqual(m.zero, v1)
        v2 = parser.parse(l([5,6,7,8,9]), 'bye')
        self.assertEqual(good(l([8,9]), 'bye', 47), v2)
        v3 = parser.parse(l([5,6]), 'goodbye')
        self.assertEqual(m.zero, v3)
    
    def testOptional(self):
        parser = c.optional(lit1(3), 'blargh')
        v1 = parser.parse(l([1,2,3]), 'hi')
        self.assertEqual(good(l([1,2,3]), 'hi', 'blargh'), v1)
        v2 = parser.parse(l([3,2,1]), 'bye')
        self.assertEqual(good(l([2,1]), 'bye', 3), v2)
    
    def testSeq2R(self):
        val = c.seq2R(lit1(2), lit1(3))
        self.assertEqual(val.parse(l([4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,3,4]), {}), good(l([4]), {}, 3))
    
    def testSeq2L(self):
        val = c.seq2L(lit1(2), lit1(3))
        self.assertEqual(val.parse(l([4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,3,4]), {}), good(l([4]), {}, 2))
    
    def testLookahead(self):
        parser = c.seq2L(lit1(2), c.lookahead(lit1(3)))
        self.assertEqual(good(l([3,4,5]), None, 2), parser.parse(l([2,3,4,5]), None))
        self.assertEqual(m.zero, parser.parse(l([2,4,5]), None))
        self.assertEqual(m.zero, parser.parse(l([3,4,5]), None))
    
    def testNot0(self):
        val = c.not0(lit1(2))
        self.assertEqual(val.parse(l([2,3,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,4,5]), {}), good(l([3,4,5]), {}, None))
    
    def testCommit(self):
        val = c.commit('bag-agg', lit1(2))
        self.assertEqual(val.parse(l([2,3,4]), 'hi'), good(l([3,4]), 'hi', 2))
        self.assertEqual(val.parse(l([3,4,5]), 'hi'), m.error('bag-agg'))
    
    def testZero(self):
        self.assertEqual(m.zero, c.zero.parse(None, None))
    
    def testGet(self):
        self.assertEqual(c.get.parse('abc', {}), good('abc', {}, 'abc'))
    
    def testGetState(self):
        self.assertEqual(good('abc', 123, 123), c.getState.parse('abc', 123))
<commit_msg>new test case: optional combinator, no default value<commit_after>from .. import combinators as c
from .. import maybeerror as me
import unittest as u


m = me.MaybeError

l = c.ConsList

def good(rest, state, result):
    return m.pure({'rest': rest, 'state': state, 'result': result})

iz1 = c.basic
(item1, lit1, sat1, not11, str1, oneOf1) = iz1.item, iz1.literal, iz1.satisfy, iz1.not1, iz1.string, iz1.oneOf
iz2 = c.position
(item2, lit2, sat2, not12, str2, oneOf2) = iz2.item, iz2.literal, iz2.satisfy, iz2.not1, iz2.string, iz2.oneOf
iz3 = c.count
(item3, lit3, sat3, not13, str3, oneOf3) = iz3.item, iz3.literal, iz3.satisfy, iz3.not1, iz3.string, iz3.oneOf



class BasicTokens(u.TestCase):

    def testItemBasic(self):
        self.assertEqual(m.zero, item1.parse(l(''), None))
        self.assertEqual(good(l('bcdef'), None, 'a'), item1.parse(l('abcdef'), None))

    def testLiteral(self):
        val = lit1(3)
        self.assertEqual(val.parse(l([3,4,5]), {}), good(l([4,5]), {}, 3))
        self.assertEqual(val.parse(l([4,5]), {}), m.zero)
    
    def testSatisfy(self):
        v1 = sat1(lambda x: x > 3).parse(l([1,2,3]), 'bye')
        self.assertEqual(m.zero, v1)
        v2 = sat1(lambda x: x < 3).parse(l([1,2,3]), 'hi')
        self.assertEqual(good(l([2,3]), 'hi', 1), v2)
    
    def testString(self):
        parser = str1('abc')
        v1 = parser.parse(l('abcdef'), None)
        self.assertEqual(good(l('def'), None, 'abc'), v1)
        v2 = parser.parse(l('abdef'), None)
        self.assertEqual(m.zero, v2)
    
    def testNot1(self):
        val = not11(lit1(2))
        self.assertEqual(val.parse(l([2,3,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,4,5]), {}), good(l([4,5]), {}, 3))
    
    def testOneOf(self):
        p = oneOf1('abc')
        self.assertEqual(p.parse(l('cqrs'), None), good(l('qrs'), None, 'c'))
        self.assertEqual(p.parse(l('aqrs'), None), good(l('qrs'), None, 'a'))
        self.assertEqual(p.parse(l('dqrs'), None), m.zero)



class PositionTokens(u.TestCase):

    def testItemPosition(self):
        self.assertEqual(m.zero, item2.parse(l(''), (1,1)))
        self.assertEqual(good(l('bcdef'), (1,2), 'a'), item2.parse(l('abcdef'), (1,1)))
        self.assertEqual(good(l('bcdef'), (2,1), '\n'), item2.parse(l('\nbcdef'), (1,1)))

    def testLiteral(self):
        val = lit2('3')
        self.assertEqual(val.parse(l('345'), (3,8)), good(l('45'), (3,9), '3'))
        self.assertEqual(val.parse(l('45'), (3,8)), m.zero)
    
    def testSatisfy(self):
        v1 = sat2(lambda x: int(x) > 3).parse(l('123'), (2,2))
        self.assertEqual(m.zero, v1)
        v2 = sat2(lambda x: int(x) < 3).parse(l('123'), (2,2))
        self.assertEqual(good(l('23'), (2,3), '1'), v2)
    
    def testString(self):
        parser = str2('abc')
        v1 = parser.parse(l('abcdef'), (4,3))
        self.assertEqual(good(l('def'), (4,6), 'abc'), v1)
        v2 = parser.parse(l('abdef'), (4,3))
        self.assertEqual(m.zero, v2)
    
    def testNot1(self):
        val = not12(lit2('2'))
        self.assertEqual(val.parse(l('234'), (1,1)), m.zero)
        self.assertEqual(val.parse(l('345'), (1,1)), good(l('45'), (1,2), '3'))
    
    def testOneOf(self):
        p = oneOf2('abc')
        self.assertEqual(p.parse(l('cqrs'), (3,4)), good(l('qrs'), (3,5), 'c'))
        self.assertEqual(p.parse(l('aqrs'), (8,1)), good(l('qrs'), (8,2), 'a'))
        self.assertEqual(p.parse(l('dqrs'), (2,2)), m.zero)



class CountTokens(u.TestCase):

    def testItemPosition(self):
        self.assertEqual(m.zero, item3.parse(l(''), 8))
        self.assertEqual(good(l('bcdef'), 6, 'a'), item3.parse(l('abcdef'), 5))
        self.assertEqual(good(l('bcdef'), 101, '\n'), item3.parse(l('\nbcdef'), 100))

    def testLiteral(self):
        val = lit3('3')
        self.assertEqual(val.parse(l('345'), 8), good(l('45'), 9, '3'))
        self.assertEqual(val.parse(l('45'), 8), m.zero)
    
    def testSatisfy(self):
        v1 = sat3(lambda x: int(x) > 3).parse(l('123'), 22)
        self.assertEqual(m.zero, v1)
        v2 = sat3(lambda x: int(x) < 3).parse(l('123'), 22)
        self.assertEqual(good(l('23'), 23, '1'), v2)
    
    def testString(self):
        parser = str3('abc')
        v1 = parser.parse(l('abcdef'), 43)
        self.assertEqual(good(l('def'), 46, 'abc'), v1)
        v2 = parser.parse(l('abdef'), 43)
        self.assertEqual(m.zero, v2)
    
    def testNot1(self):
        val = not13(lit3('2'))
        self.assertEqual(val.parse(l('234'), 61), m.zero)
        self.assertEqual(val.parse(l('345'), 61), good(l('45'), 62, '3'))
    
    def testOneOf(self):
        p = oneOf3('abc')
        self.assertEqual(p.parse(l('cqrs'), 4), good(l('qrs'), 5, 'c'))
        self.assertEqual(p.parse(l('aqrs'), 8), good(l('qrs'), 9, 'a'))
        self.assertEqual(p.parse(l('dqrs'), 7), m.zero)



class TestParser(u.TestCase):
    
    def testFmap(self):
        f = lambda x: x + 7
        v1 = c.fmap(f, c.pure(3)).parse('ab', 81)
        self.assertEqual(good('ab', 81, 10), v1)
        v2 = c.fmap(f, c.zero).parse('ab', 81)
        self.assertEqual(m.zero, v2)
        v3 = c.fmap(f, c.error('oops')).parse('ab', 81)
        self.assertEqual(m.error('oops'), v3)
        
    def testPure(self):
        val = c.pure(3).parse('abc', 2)
        self.assertEqual(good('abc', 2, 3), val)
    
    def testBind(self):
        two = c.bind(item1, lambda x: lit1(x))
        self.assertEqual(two.parse(l('abcde'), {}), m.zero)
        self.assertEqual(two.parse(l('aabcde'), {}), good(l('bcde'), {}, 'a'))

    def testAltBinaryRules(self):
        g1, g2, b, e, e2 = c.pure(3), c.pure('hi'), c.zero, c.error('oops'), c.error('2nd')
        r1, r3, r4 = good('abc', None, 3), m.zero, m.error('oops')
        self.assertEqual(c.alt(g1, g2).parse('abc', None), r1)
        self.assertEqual(c.alt(g1, b).parse('abc', None), r1)
        self.assertEqual(c.alt(g1, e).parse('abc', None), r1)
        self.assertEqual(c.alt(b, g1).parse('abc', None), r1)
        self.assertEqual(c.alt(b, b).parse('abc', None), r3)
        self.assertEqual(c.alt(b, e).parse('abc', None), r4)
        self.assertEqual(c.alt(e, g1).parse('abc', None), r4)
        self.assertEqual(c.alt(e, b).parse('abc', None), r4)
        self.assertEqual(c.alt(e, e2).parse('abc', None), r4)
    
    def testAltCornerCases(self):
        self.assertEqual(m.zero, 
                         c.alt().parse(l([1,2,3]), None))
        self.assertEqual(good(l([1,2,3]), None, 'h'), 
                         c.alt(c.pure('h')).parse(l([1,2,3]), None))
        self.assertEqual(m.error('oops'), 
                         c.alt(c.error('oops')).parse(l([1,2,3]), None))
        self.assertEqual(m.zero, 
                         c.alt(c.zero).parse(l([1,2,3]), None))
        p1 = c.alt(c.zero, lit1(1), lit1(2), c.error('d'))
        self.assertEqual(good(l([3,4]), None, 1), p1.parse(l([1,3,4]), None))
        self.assertEqual(good(l([3,4]), None, 2), p1.parse(l([2,3,4]), None))
        self.assertEqual(m.error('d'), p1.parse(l([3,3,4]), None))
    
    def testError(self):
        v1 = c.error('uh-oh').parse('abc', 123)
        self.assertEqual(m.error('uh-oh'), v1)
    
    def testCatchError(self):
        f1 = lambda e: c.pure(3)
        f2 = lambda e: c.error('dead again')
        # error -> good -- resumes parsing with tokens and state from before the error occurred
        self.assertEqual(good('123', [2,4], 3), c.catchError(f1, c.error('dead 1')).parse('123', [2, 4]))
        # good -> good (unaffected by this combinator)
        self.assertEqual(good('123', [2,4], 18), c.catchError(f1, c.pure(18)).parse('123', [2,4]))
        # error -> error
        self.assertEqual(m.error('dead again'), c.catchError(f2, c.error('dead 1')).parse('123', [2,4]))
        # good -> error is not possible with this combinator
        
    def testMapError(self):
        f = len
        v1 = c.mapError(f, c.error('abcdef')).parse('123abc', None)
        self.assertEqual(m.error(6), v1)
        v2 = c.mapError(f, c.zero).parse('123abc', None)
        self.assertEqual(m.zero, v2)
        v3 = c.mapError(f, c.pure(82)).parse('123abc', None)
        self.assertEqual(good('123abc', None, 82), v3)        

    def testPut(self):
        val = c.put('xyz')
        self.assertEqual(val.parse('abc', []), good('xyz', [], None))
    
    def testPutState(self):
        v1 = c.putState(29).parse('abc123', 2)
        self.assertEqual(good('abc123', 29, None), v1)
    
    def testUpdateState(self):
        v1 = c.updateState(lambda x: x * 4).parse('abc', 18)
        self.assertEqual(good('abc', 72, None), v1)
        
    def testCheck(self):
        val = c.check(lambda x: len(x) > 3, c.get)
        self.assertEqual(val.parse('abcde', []), good('abcde', [], 'abcde'))
        self.assertEqual(val.parse('abc', []), m.zero)
    
    def testMany0(self):
        val = c.many0(lit1(3))
        self.assertEqual(val.parse(l([4,4,4]), {}), good(l([4,4,4]), {}, []))
        self.assertEqual(val.parse(l([3,3,4,5]), {}), good(l([4,5]), {}, [3,3]))
    
    def testMany1(self):
        val = c.many1(lit1(3))
        self.assertEqual(val.parse(l([4,4,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,3,4,5]), {}), good(l([4,5]), {}, [3,3]))
    
    def testSeq(self):
        val = c.seq(item1, lit1(2), lit1(8))
        self.assertEqual(val.parse(l([3,2,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,2,8,16]), {}), good(l([16]), {}, [3,2,8]))
    
    def testApp(self):
        parser = c.app(lambda x,y,z: x + y * z, item1, sat1(lambda x: x > 2), item1)
        v1 = parser.parse(l([1,2,3,4,5]), 'hi')
        self.assertEqual(m.zero, v1)
        v2 = parser.parse(l([5,6,7,8,9]), 'bye')
        self.assertEqual(good(l([8,9]), 'bye', 47), v2)
        v3 = parser.parse(l([5,6]), 'goodbye')
        self.assertEqual(m.zero, v3)
    
    def testOptional(self):
        parser = c.optional(lit1(3), 'blargh')
        v1 = parser.parse(l([1,2,3]), 'hi')
        self.assertEqual(good(l([1,2,3]), 'hi', 'blargh'), v1)
        v2 = parser.parse(l([3,2,1]), 'bye')
        self.assertEqual(good(l([2,1]), 'bye', 3), v2)
    
    def testOptionalNoValue(self):
        p = c.optional(lit1(3))
        v1 = p.parse(l([3,2,1]), None)
        self.assertEqual(good(l([2,1]), None, 3), v1)
        v2 = p.parse(l([1,2,3]), None)
        self.assertEqual(good(l([1,2,3]), None, None), v2)
    
    def testSeq2R(self):
        val = c.seq2R(lit1(2), lit1(3))
        self.assertEqual(val.parse(l([4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,3,4]), {}), good(l([4]), {}, 3))
    
    def testSeq2L(self):
        val = c.seq2L(lit1(2), lit1(3))
        self.assertEqual(val.parse(l([4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,4,5]), {}), m.zero)
        self.assertEqual(val.parse(l([2,3,4]), {}), good(l([4]), {}, 2))
    
    def testLookahead(self):
        parser = c.seq2L(lit1(2), c.lookahead(lit1(3)))
        self.assertEqual(good(l([3,4,5]), None, 2), parser.parse(l([2,3,4,5]), None))
        self.assertEqual(m.zero, parser.parse(l([2,4,5]), None))
        self.assertEqual(m.zero, parser.parse(l([3,4,5]), None))
    
    def testNot0(self):
        val = c.not0(lit1(2))
        self.assertEqual(val.parse(l([2,3,4]), {}), m.zero)
        self.assertEqual(val.parse(l([3,4,5]), {}), good(l([3,4,5]), {}, None))
    
    def testCommit(self):
        val = c.commit('bag-agg', lit1(2))
        self.assertEqual(val.parse(l([2,3,4]), 'hi'), good(l([3,4]), 'hi', 2))
        self.assertEqual(val.parse(l([3,4,5]), 'hi'), m.error('bag-agg'))
    
    def testZero(self):
        self.assertEqual(m.zero, c.zero.parse(None, None))
    
    def testGet(self):
        self.assertEqual(c.get.parse('abc', {}), good('abc', {}, 'abc'))
    
    def testGetState(self):
        self.assertEqual(good('abc', 123, 123), c.getState.parse('abc', 123))

{'fraction_non_alphanumeric': 0.2285376960701636, 'fraction_numerical': 0.05355034575813797, 'mean_word_length': 2.797758206565252, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1533472', 'n_tokens_mistral': 9485, 'n_tokens_neox': 9096, 'n_words': 1569}