<commit_before>import sys
import os
import glob
import StringIO
import unittest
import new

def parseTestcase(testString):
    testString = testString.split("\n")
    try:
        assert testString[0] == "#data"
    except:
        raise
    input = []
    output = []
    errors = []
    currentList = input
    for line in testString:
        if line and line[0] != "#":
            if currentList is output:
                assert line[0] == "|"
                currentList.append(line[2:])
                # XXX the line might not start with a "|" if it's a
                # continuation line, e.g. if a text node contained a linefeed
            else:
                currentList.append(line)
        elif line == "#errors":
            currentList = errors
        elif line == "#document":
            currentList = output
    return "\n".join(input), "\n".join(output), errors

def convertTreeDump(treedump):
    """convert the output of str(document) to the format used in the testcases"""
    treedump = treedump.split("\n")
    rv = []
    for line in treedump:
        if line.startswith("#document"):
            pass
        else:
            rv.append(line[3:])
    return "\n".join(rv)

class TestCase(unittest.TestCase):
    def runParserTest(self, input, output, errors):
        import parser
        #XXX - move this out into the setup function
        #concatenate all consecutive character tokens into a single token
        p = parser.HTMLParser()
        document = p.parse(StringIO.StringIO(input))
        errorMsg = "\n".join(["\n\nExpected:", output, "\nRecieved:", <commit_msg>simplify convertTreeDump in test_parser.py as well<commit_after>import sys
import os
import glob
import StringIO
import unittest
import new

def parseTestcase(testString):
    testString = testString.split("\n")
    try:
        assert testString[0] == "#data"
    except:
        raise
    input = []
    output = []
    errors = []
    currentList = input
    for line in testString:
        if line and line[0] != "#":
            if currentList is output:
                assert line[0] == "|"
                currentList.append(line[2:])
                # XXX the line might not start with a "|" if it's a
                # continuation line, e.g. if a text node contained a linefeed
            else:
                currentList.append(line)
        elif line == "#errors":
            currentList = errors
        elif line == "#document":
            currentList = output
    return "\n".join(input), "\n".join(output), errors

def convertTreeDump(treedump):
    """convert the output of str(document) to the format used in the testcases"""
    treedump = treedump.split("\n")[1:]
    rv = []
    for line in treedump:
        rv.append(line[3:])
    return "\n".join(rv)

class TestCase(unittest.TestCase):
    def runParserTest(self, input, output, errors):
        import parser
        #XXX - move this out into the setup function
        #concatenate all consecutive character tokens into a single token
        p = parser.HTMLParser()
        document = p.parse(StringIO.StringIO(input))
        errorMsg = "\n".join(["\n\nExpected:", output, "\nRecieved:", 
{'fraction_non_alphanumeric': 0.11261402956904687, 'fraction_numerical': 0.0034602076124567475, 'mean_word_length': 2.1927710843373496, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3997520', 'n_tokens_mistral': 906, 'n_tokens_neox': 886, 'n_words': 283}