<commit_before>import sys
import glob
import StringIO

import simplejson

#Allow us to import the parent module
sys.path.insert(0, "../")

import tokenizer

class TokenizerTestParser(object):
    def parse(self, stream, innerHTML=False):
        """Stream should be a stream of unicode bytes. Character encoding
        issues have not yet been dealt with."""

        self.outputTokens = []        

        self.tokenizer = tokenizer.HTMLTokenizer(self)
        self.tokenizer.tokenize(stream)
        
        return self.outputTokens

    def processDoctype(self, name, error):
        self.outputTokens.append([u"DOCTYPE", name, error])

    def processStartTag(self, name, attributes):
        self.outputTokens.append([u"StartTag", name, attributes])

    def processEndTag(self, name):
        self.outputTokens.append([u"EndTag", name])

    def processComment(self, data):
        self.outputTokens.append([u"Comment", data])

    def processCharacter(self, data):
        self.outputTokens.append([u"Character", data])
        
    def processEOF(self):
        pass

    def parseError(self):
        self.outputTokens.append(u"ParseError")

    def atheistParseError(self):
        """This error is not an error"""
        self.outputTokens.append(u"AtheistParseError")

def concatanateCharacterTokens(tokens):
    outputTokens = []
    for token in tokens:
        if not "ParseError" in token and token[0] == "Character":
            if (outputTokens and not "ParseError" in outputTokens[-1] and 
                outputTokens[-1][0] == "Character"):
                outputTokens[-1][1] += token[1]
            else:
                outputTokens.append(token)
        else:
            outputTokens.append(token)
    return outputTokens

def tokensMatch(expectedTokens, recievedTokens):
    """Test whether the test has passed or failed

    For brevity in the tests, the test has passed if the sequence of expected
    tokens appears anywhere in the sequqnce of returned tokens.
    """
    return expectedTokens == recievedTokens
    for i, token in enumerate(recievedTokens):
        if expectedTokens[0] == token:
            if (len(expectedTokens) <= len(recievedTokens[i:]) and
                recievedTokens[i:i+len(expectedTokens)]):
                return True
    return False
        

def test_tokenizer():
    for filename in glob.glob('tokenizer/*.test'):
        tests = simplejson.load(file(filename))
        for test in tests['tests']:
            yield (runTokenizerTest, test['description'], test['input'], 
                   test['output'])

def runTokenizerTest(description, input, output):
    #XXX - move this out into the setup function
    #concatanate all consecutive character tokens into a single token
    output = concatanateCharacterTokens(output)
    parser = TokenizerTestParser()
    tokens = parser.parse(StringIO.StringIO(input))
    tokens = concatanateCharacterTokens(tokens)
    print "Got", tokens, "expected", output
    assert tokensMatch(tokens, output)

def main():
    failed = 0
    tests = 0
    for func, desc, input, output in test_tokenizer():
        tests += 1
        try:
            func(desc, input, output)
        except AssertionError:
            print "Failed test %s"%(desc,)
            parser = TokenizerTestParser()
            tokens = parser.parse(StringIO.StringIO(input))
            print "Got", tokens, "expected", output
            failed +=1
    print "Ran %i tests, failed %i"%(tests, failed)

if __name__ == "__main__":
    main()<commit_msg>Sam Ruby's patch to use unittest rather than nose as the test framework, and preform a few other bits of cleanup. Patch to documentation coming up.<commit_after>import sys
import os
import glob
import StringIO
import unittest
import new

import simplejson

#Allow us to import the parent module
os.chdir(os.path.split(os.path.abspath(__file__))[0])
sys.path.insert(0, os.path.abspath(os.pardir))

import tokenizer

class TokenizerTestParser(object):
    def parse(self, stream, innerHTML=False):
        """Stream should be a stream of unicode bytes. Character encoding
        issues have not yet been dealt with."""

        self.outputTokens = []        

        self.tokenizer = tokenizer.HTMLTokenizer(self)
        self.tokenizer.tokenize(stream)
        
        return self.outputTokens

    def processDoctype(self, name, error):
        self.outputTokens.append([u"DOCTYPE", name, error])

    def processStartTag(self, name, attributes):
        self.outputTokens.append([u"StartTag", name, attributes])

    def processEndTag(self, name):
        self.outputTokens.append([u"EndTag", name])

    def processComment(self, data):
        self.outputTokens.append([u"Comment", data])

    def processCharacter(self, data):
        self.outputTokens.append([u"Character", data])
        
    def processEOF(self):
        pass

    def parseError(self):
        self.outputTokens.append(u"ParseError")

    def atheistParseError(self):
        """This error is not an error"""
        self.outputTokens.append(u"AtheistParseError")

def concatenateCharacterTokens(tokens):
    outputTokens = []
    for token in tokens:
        if not "ParseError" in token and token[0] == "Character":
            if (outputTokens and not "ParseError" in outputTokens[-1] and 
                outputTokens[-1][0] == "Character"):
                outputTokens[-1][1] += token[1]
            else:
                outputTokens.append(token)
        else:
            outputTokens.append(token)
    return outputTokens

def tokensMatch(expectedTokens, recievedTokens):
    """Test whether the test has passed or failed

    For brevity in the tests, the test has passed if the sequence of expected
    tokens appears anywhere in the sequence of returned tokens.
    """
    return expectedTokens == recievedTokens
    for i, token in enumerate(recievedTokens):
        if expectedTokens[0] == token:
            if (len(expectedTokens) <= len(recievedTokens[i:]) and
                recievedTokens[i:i+len(expectedTokens)]):
                return True
    return False
        

class TestCase(unittest.TestCase):
    def runTokenizerTest(self, input, output):
        #XXX - move this out into the setup function
        #concatenate all consecutive character tokens into a single token
        output = concatenateCharacterTokens(output)
        parser = TokenizerTestParser()
        tokens = parser.parse(StringIO.StringIO(input))
        tokens = concatenateCharacterTokens(tokens)
        self.assertTrue(tokensMatch(tokens, output))

def test_tokenizer():
    for filename in glob.glob('tokenizer/*.test'):
        tests = simplejson.load(file(filename))
        for test in tests['tests']:
            yield (TestCase.runTokenizerTest, test['description'],
                   test['input'], test['output'])

def main():
    failed = 0
    tests = 0
    for func, desc, input, output in test_tokenizer():
        tests += 1
        testName = 'test%d' % tests
        testFunc = lambda self, method=func, input=input, output=output: \
            method(self, input, output)
        testFunc.__doc__ = desc
        instanceMethod = new.instancemethod(testFunc, None, TestCase)
        setattr(TestCase, testName, instanceMethod)
    unittest.main()

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.10163934426229508, 'fraction_numerical': 0.003551912568306011, 'mean_word_length': 2.797199170124481, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5289971', 'n_tokens_mistral': 2020, 'n_tokens_neox': 2015, 'n_words': 580}