<commit_before>import json,time
import codecs

def edits1(word):
   splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
   deletes = [a + b[1:] for a, b in splits if b]
   return set(deletes)

def edits2(word):
    return set(e2 for e1 in edits1(word) for e2 in edits1(e1))


def known(words,NWORDS): return set(w for w in words if w in NWORDS)

def correct(word,dicts):
  word = word.lower()
  # print dicts
  if known([word],dicts["city"]):
    return "city"
  if len(word) >= 5:
    if known(edits1(word),dicts["city"]):
      return "city"
  
  return "False"



# print correct("losangeles",jf)<commit_msg>tagging function<commit_after>import json
import time
def edits1(word):
   splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
   deletes = [a + b[1:] for a, b in splits if b]
   return set(deletes)

def edits2(word):
    return set(e2 for e1 in edits1(word) for e2 in edits1(e1))


def known(words,NWORDS): return set(w for w in words if w in NWORDS)

def tag(word,dicts,key):
  word = word.lower()
  if known([word],dicts[key]):
    return key
  elif len(word) >= 5:
    if known(edits1(word),dicts[key]):
      return key
  else:
    return None


# jf = json.load(open("dicts/tagging_dict.json"))
# print "dict loaded"
# start_time = time.clock()
# print tag("los angele",jf,"city")
# print tag("losangele2",jf,"city")
# print tag("california",jf,"state")
# print tag("calnifornib",jf,"state")
# process_time = str((time.clock() - start_time)*1000)
# print process_time
{'fraction_non_alphanumeric': 0.16023936170212766, 'fraction_numerical': 0.01928191489361702, 'mean_word_length': 4.225694444444445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7319380', 'n_tokens_mistral': 601, 'n_tokens_neox': 578, 'n_words': 160}