<commit_before>import sys
sys.path.append( '.' )

import redis

from patXML import XMLPatent
from patXML import uniasc
from fwork  import *
import os, datetime, re

flder = '/var/share/patentdata/patents/2010'

t1 = datetime.datetime.now()

#get a listing of all files within the directory that follow the naming pattern
files = [x for x in os.listdir(flder) if re.match(r"ipg\d{6}.xml", x, re.I)!=None]

print "Total files: %d" % (len(files))

#sys.exit()

tables = ["assignee", "citation", "class", "inventor", "patent", "patdesc", "lawyer", "sciref", "usreldoc"]
total_count = 0
total_patents = 0

r = redis.StrictRedis(host='localhost', port=6379, db=0)
#r = redis.Redis(host='localhost', port=6379, db=0)

for filenum, filename in enumerate(files):
    print " > Regular Expression: %s" % filename
    XMLs = re.findall(
            r"""
                ([<][?]xml[ ]version.*?[>]       #all XML starts with ?xml
                .*?
                [<][/]us[-]patent[-]grant[>])    #and here is the end tag
             """,
            open(flder+"/"+files[filenum]).read(), re.I + re.S + re.X)
    print "   - Total Patents: %d" % (len(XMLs))


    for i, x in enumerate(XMLs):
        r.set(i,x)
#
#
#
#    xmllist = []
#    count = 0
#    patents = 0
#    for i, x in enumerate(XMLs):
#        try:
#            xmllist.append(XMLPatent(x))
#            patents += 1
#        except Exception as inst:
#            print type(inst)
#            print "  - Error: %s (%d)  %s" % (filename, i, x[175:200])
#            count += 1
#    print "   - number of patents:", len(xmllist), datetime.datetime.now()-t1
#    print "   - number of errors: " ,count
#    total_count += count
#    total_patents += patents
#    print "   -", datetime.datetime.now()-t1
#    print "   - total errors: ", total_count
#    print "   - total patents: ", total_patents
#
<commit_msg>removed superfluous comments from redistest<commit_after>import sys
sys.path.append( '.' )

import redis

from patXML import XMLPatent
from patXML import uniasc
from fwork  import *
import os, datetime, re

flder = '/var/share/patentdata/patents/2010'

t1 = datetime.datetime.now()

#get a listing of all files within the directory that follow the naming pattern
files = [x for x in os.listdir(flder) if re.match(r"ipg\d{6}.xml", x, re.I)!=None]

print "Total files: %d" % (len(files))

#sys.exit()

tables = ["assignee", "citation", "class", "inventor", "patent", "patdesc", "lawyer", "sciref", "usreldoc"]
total_count = 0
total_patents = 0

r = redis.StrictRedis(host='localhost', port=6379, db=0)
#r = redis.Redis(host='localhost', port=6379, db=0)

for filenum, filename in enumerate(files):
    print " > Regular Expression: %s" % filename
    XMLs = re.findall(
            r"""
                ([<][?]xml[ ]version.*?[>]       #all XML starts with ?xml
                .*?
                [<][/]us[-]patent[-]grant[>])    #and here is the end tag
             """,
            open(flder+"/"+files[filenum]).read(), re.I + re.S + re.X)
    print "   - Total Patents: %d" % (len(XMLs))


    for i, x in enumerate(XMLs):
        r.set(i,x)

{'fraction_non_alphanumeric': 0.16618543471286493, 'fraction_numerical': 0.015399422521655439, 'mean_word_length': 2.698695136417556, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1501207', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1104, 'n_words': 295}