<commit_before># -*- coding: utf-8 -*-
import os, sys,csv
import traceback
reload(sys)
sys.setdefaultencoding('utf-8')

print "Note - run from ./solr/ folder" 
print "Ingesting filter files"

GAZ_CONF = './etc/gazetteer'
TAX_CONF = './etc/taxcat'
# The resulting name filter file:
output = os.path.join(GAZ_CONF, 'filters/person-name-filter.txt')
nonpersons = os.path.join(TAX_CONF, 'non-person-names.txt')

# All final names
names = set([])

def is_comment(s):
  if not s: return False
  if not s.strip(): return False
  return s.strip()[0] == '#'


# USGS/Census top 1000 surnames
#  To counter-act these names, add entries to  conf/inclusions/adhoc-places.txt
#
# Formerly manual download: filters/census/top1000-surnames-cy2010.csv
fpath =   os.path.join(GAZ_CONF, 'filters/census/Names_2010Census.csv')
TOP_N=1500
f1 =  os.path.abspath( fpath )
if os.path.exists(f1):
    print "\tParse ", f1
    with open(f1, 'rb') as fh:
        counter=0
        csvh = csv.reader(fh, delimiter=',')
        for row in csvh:
            if not is_comment(row[0]):
                names.add(unicode(row[0]).strip().lower())
                counter += 1
                if counter > TOP_N: 
                    print ("Found top N names")
                    break
else:
   print "Census data not present:", fpath

#
fpath = os.path.join(GAZ_CONF, 'filters/census/dist.male.first')
f1 =  os.path.abspath( fpath )
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    # csvh = csv.reader(fh, delimiter='')
    for line in fh:
        row = line.split() 
        name = row[0].lower()
        if not is_comment(name) and len(name)>2:
            names.add(unicode(name))

    fh.close()
else:
   print "Census data not present:", fpath

fpath = os.path.join(GAZ_CONF, 'filters/census/dist.female.first')
f1 =  os.path.abspath( fpath )
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    # csvh = csv.reader(fh, delimiter='')
    for line in fh:
        row = line.split() 
        name = row[0].lower()
        if not is_comment(name) and len(name)>2:
            names.add(unicode(name))

    fh.close()
else:
   print "Census data not present:", fpath
    
# Other useful names to use as not-place entries.
#
f1 = os.path.abspath(os.path.join(GAZ_CONF, 'filters/exclude-adhoc-names.txt'))
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    for row in fh:
        if not is_comment(row):
            names.add(unicode(row).strip().lower())
    fh.close()


# a list of valid known places.  Remove known places from name filter.
# 
f1 = os.path.abspath( os.path.join(GAZ_CONF, 'filters/include-adhoc-places.txt' ))
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    for row in fh:
        if not is_comment(row):
            k = unicode(row).strip().lower()
            if k in names: names.remove(k)
    fh.close()


non_person_names = set([])
with open(nonpersons, 'rb') as fh:
    for nm in fh:
        non_person_names.add(nm.strip().lower())

print "Found non-person names:", len(non_person_names)

#
if names:
    for nm in non_person_names:
        names.remove(nm)

    fh = open(output, 'wb')
    fh.write('# Generated File:  census surnames + exclusions - inclusions\n')
    name_list = sorted(list(names))
    for n in name_list:
        fh.write(n)
        fh.write('\n')
    fh.close()

print "\tWahoo. Done"
print "\tResulting names filter file is at", os.path.abspath(output)<commit_msg>set remove requires item to be in set<commit_after># -*- coding: utf-8 -*-
import os, sys,csv
import traceback
reload(sys)
sys.setdefaultencoding('utf-8')

print "Note - run from ./solr/ folder" 
print "Ingesting filter files"

GAZ_CONF = './etc/gazetteer'
TAX_CONF = './etc/taxcat'
# The resulting name filter file:
output = os.path.join(GAZ_CONF, 'filters/person-name-filter.txt')
nonpersons = os.path.join(TAX_CONF, 'non-person-names.txt')

# All final names
names = set([])

def is_comment(s):
  if not s: return False
  if not s.strip(): return False
  return s.strip()[0] == '#'


# USGS/Census top 1000 surnames
#  To counter-act these names, add entries to  conf/inclusions/adhoc-places.txt
#
# Formerly manual download: filters/census/top1000-surnames-cy2010.csv
fpath =   os.path.join(GAZ_CONF, 'filters/census/Names_2010Census.csv')
TOP_N=1500
f1 =  os.path.abspath( fpath )
if os.path.exists(f1):
    print "\tParse ", f1
    with open(f1, 'rb') as fh:
        counter=0
        csvh = csv.reader(fh, delimiter=',')
        for row in csvh:
            if not is_comment(row[0]):
                names.add(unicode(row[0]).strip().lower())
                counter += 1
                if counter > TOP_N: 
                    print ("Found top N names")
                    break
else:
   print "Census data not present:", fpath

#
fpath = os.path.join(GAZ_CONF, 'filters/census/dist.male.first')
f1 =  os.path.abspath( fpath )
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    # csvh = csv.reader(fh, delimiter='')
    for line in fh:
        row = line.split() 
        name = row[0].lower()
        if not is_comment(name) and len(name)>2:
            names.add(unicode(name))

    fh.close()
else:
   print "Census data not present:", fpath

fpath = os.path.join(GAZ_CONF, 'filters/census/dist.female.first')
f1 =  os.path.abspath( fpath )
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    # csvh = csv.reader(fh, delimiter='')
    for line in fh:
        row = line.split() 
        name = row[0].lower()
        if not is_comment(name) and len(name)>2:
            names.add(unicode(name))

    fh.close()
else:
   print "Census data not present:", fpath
    
# Other useful names to use as not-place entries.
#
f1 = os.path.abspath(os.path.join(GAZ_CONF, 'filters/exclude-adhoc-names.txt'))
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    for row in fh:
        if not is_comment(row):
            names.add(unicode(row).strip().lower())
    fh.close()


# a list of valid known places.  Remove known places from name filter.
# 
f1 = os.path.abspath( os.path.join(GAZ_CONF, 'filters/include-adhoc-places.txt' ))
if os.path.exists(f1):
    print "\tParse ", f1
    fh = open(f1, 'rb')
    for row in fh:
        if not is_comment(row):
            k = unicode(row).strip().lower()
            if k in names: names.remove(k)
    fh.close()


non_person_names = set([])
with open(nonpersons, 'rb') as fh:
    for nm in fh:
        non_person_names.add(nm.strip().lower())

print "Found non-person names:", len(non_person_names)

#
if names:
    for nm in non_person_names:
        if nm in names:
            names.remove(nm)

    fh = open(output, 'wb')
    fh.write('# Generated File:  census surnames + exclusions - inclusions\n')
    name_list = sorted(list(names))
    for n in name_list:
        fh.write(n)
        fh.write('\n')
    fh.close()

print "\tWahoo. Done"
print "\tResulting names filter file is at", os.path.abspath(output)
{'fraction_non_alphanumeric': 0.1462129840546697, 'fraction_numerical': 0.014521640091116172, 'mean_word_length': 2.639896373056995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6067943', 'n_tokens_mistral': 2654, 'n_tokens_neox': 2511, 'n_words': 642}