<commit_before>import sys
import os
import glob
import StringIO
import unittest
import new

import simplejson

#Allow us to import the parent module
os.chdir(os.path.split(os.path.abspath(__file__))[0])
sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, "src")))

from tokenizer import HTMLTokenizer

class TokenizerTestParser(object):
    def parse(self, stream, innerHTML=False):
        self.outputTokens = []

        self.tokenizer = HTMLTokenizer(stream)

        for token in self.tokenizer:
            getattr(self, 'process%s' % token["type"])(token)

        return self.outputTokens

    def processDoctype(self, token):
        self.outputTokens.append([u"DOCTYPE", token["name"], token["data"]])

    def processStartTag(self, token):
        self.outputTokens.append([u"StartTag", token["name"], token["data"]])

    def processEndTag(self, token):
        self.outputTokens.append([u"EndTag", token["name"]])

    def processComment(self, token):
        self.outputTokens.append([u"Comment", token["data"]])

    def processCharacters(self, token):
        self.outputTokens.append([u"Character", token["data"]])

    def processEOF(self, token):
        pass

    def processParseError(self, token):
        self.outputTokens.append(u"ParseError")

    def processAtheistParseError(self, token):
        """This error is not an error"""
        self.outputTokens.append(u"AtheistParseError")

def concatenateCharacterTokens(tokens):
    outputTokens = []
    for token in tokens:
        if not "ParseError" in token and token[0] == "Character":
            if (outputTokens and not "ParseError" in outputTokens[-1] and
                outputTokens[-1][0] == "Character"):
                outputTokens[-1][1] += token[1]
            else:
                outputTokens.append(token)
        else:
            outputTokens.append(token)
    return outputTokens

def tokensMatch(expectedTokens, recievedTokens):
    """Test whether the test has passed or failed<commit_msg>fix two tests, try fixing test_tokenizer a bit<commit_after>import sys
import os
import glob
import StringIO
import unittest
import new

import simplejson

#Allow us to import the parent module
os.chdir(os.path.split(os.path.abspath(__file__))[0])
sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, "src")))

from tokenizer import HTMLTokenizer

class TokenizerTestParser(object):
    def parse(self, stream, innerHTML=False):
        self.outputTokens = []

        self.tokenizer = HTMLTokenizer(stream)

        for token in self.tokenizer:
            getattr(self, 'process%s' % token["type"])(token)

        return self.outputTokens

    def processDoctype(self, token):
        self.outputTokens.append([u"DOCTYPE", token["name"], token["data"]])

    def processStartTag(self, token):
        self.outputTokens.append([u"StartTag", token["name"], token["data"]])

    def processEndTag(self, token):
        self.outputTokens.append([u"EndTag", token["name"]])

    def processComment(self, token):
        self.outputTokens.append([u"Comment", token["data"]])

    def processSpaceCharacters(self, token):
        self.outputTokens.append([u"Character", token["data"]])
        self.processSpaceCharacters = self.processCharacters

    def processCharacters(self, token):
        self.outputTokens.append([u"Character", token["data"]])

    def processEOF(self, token):
        pass

    def processParseError(self, token):
        self.outputTokens.append(u"ParseError")

    def processAtheistParseError(self, token):
        """This error is not an error"""
        self.outputTokens.append(u"AtheistParseError")

def concatenateCharacterTokens(tokens):
    outputTokens = []
    for token in tokens:
        if not "ParseError" in token and token[0] == "Character":
            if (outputTokens and not "ParseError" in outputTokens[-1] and
                outputTokens[-1][0] == "Character"):
                outputTokens[-1][1] += token[1]
            else:
                outputTokens.append(token)
        else:
            outputTokens.append(token)
    return outputTokens

def tokensMatch(expectedTokens, recievedTokens):
    """Test whether the test has passed or failed
{'fraction_non_alphanumeric': 0.12368922783603432, 'fraction_numerical': 0.004289799809342231, 'mean_word_length': 3.331269349845201, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7285249', 'n_tokens_mistral': 1231, 'n_tokens_neox': 1234, 'n_words': 285}