<commit_before>import unittest
import sys
import imp
import os
import datetime
import re
import logging
from xml.dom.minidom import parse, parseString
from xml_patent import XMLPatent
from optparse import OptionParser

# xml fixtures
xml_file1 = 'unittest/test1.xml'
xml_file2 = 'unittest/test2.xml'
xml_file3 = 'unittest/test3.xml'
xml_file4 = 'unittest/test4.xml'
xml_file5 = 'unittest/test5.xml'
xml_file6 = 'unittest/test6.xml'
xml_file7 = 'unittest/test7.xml'
xml_file8 = 'unittest/test8.xml'
xml_file9 = 'unittest/test9.xml'
xml_file10 = 'unittest/test10.xml'

# Non working xml fixtures
xml_fileu1 = 'unittest/testu1.xml'
xml_fileu2 = 'unittest/testu2.xml'
xml_fileu3 = 'unittest/testu3.xml'

# Data structures/variables used in testing
debug = False
xml_files = []
parsed_xml = []
max_years = "2012"
max_months = "12"
max_days = "31"

dir = os.path.dirname(__file__)
folder = os.path.join(dir, 'unittest/')
xml_files = [x for x in os.listdir(folder)]

# TODO:
#Fix formatting
#Update field logic tests
# - test months not past 12, days past 31, years past 2012?
#Update xml presence tests
#Implement logging
#Profiling?

<commit_msg>Updated todo, changed os dir to only look for files with "test" in them.<commit_after>import unittest
import sys
import imp
import os
import datetime
import re
import logging
from xml.dom.minidom import parse, parseString
from xml_patent import XMLPatent
from optparse import OptionParser

# xml fixtures
xml_file1 = 'unittest/test1.xml'
xml_file2 = 'unittest/test2.xml'
xml_file3 = 'unittest/test3.xml'
xml_file4 = 'unittest/test4.xml'
xml_file5 = 'unittest/test5.xml'
xml_file6 = 'unittest/test6.xml'
xml_file7 = 'unittest/test7.xml'
xml_file8 = 'unittest/test8.xml'
xml_file9 = 'unittest/test9.xml'
xml_file10 = 'unittest/test10.xml'

# Non working xml fixtures
xml_fileu1 = 'unittest/testu1.xml'
xml_fileu2 = 'unittest/testu2.xml'
xml_fileu3 = 'unittest/testu3.xml'

# Data structures/variables used in testing
debug = False
xml_files = []
parsed_xml = []
max_years = "2012"
max_months = "12"
max_days = "31"

dir = os.path.dirname(__file__)
folder = os.path.join(dir, 'unittest/')
xml_files = [x for x in os.listdir(folder)
             if re.match(r"test.*?xml", x) != None]

# TODO:
#Update xml presence tests, make it more robust. 
#Implement logging


{'fraction_non_alphanumeric': 0.11125597566275533, 'fraction_numerical': 0.034767492394611035, 'mean_word_length': 4.994791666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 69}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4899414', 'n_tokens_mistral': 877, 'n_tokens_neox': 845, 'n_words': 223}