<commit_before>import speech_recognition as sr
import os
import time
import json
from collections import Counter
import logging, sys
import multiprocessing 
import pprint
import csv
import new_filter_function
import threading
import googleapiclient
LEVEL = logging.INFO
logging.basicConfig(stream=sys.stderr, level=LEVEL)
logging.getLogger('oauth2client.transport').setLevel(logging.ERROR)
logging.getLogger('googleapiclient.discovery').setLevel(logging.CRITICAL)  
logging.getLogger('oauth2client.client').setLevel(logging.ERROR)

pp = pprint.PrettyPrinter(indent=4)

#DEFAULT = "X" # all un-identified digits remain unknown
DEFAULT = "6" # all un-identified digits are mapped to "6"

apis = ["googleCloud", "wit", "bing", "ibm", "google", "sphinx"]

#Simple homophone mapping, taking any exact matches and returning the digit (layer one mapping)
def homophone(num):
	if num in ["one", "1", "won"]:
		return "1"
	elif num in ["two", "to", "too", "2"]:
		return "2"
	elif num in ["three", "3"]:
		return "3"
	elif num in ["four", "for", "4", "fore"]:
		return "4"
	elif num in ["five", "5"]:
		return "5"
	elif num in ["six", "6"]:
		return "6"
	elif num in ["seven", "7"]:
		return "7"
	elif num in ["eight", "ate", "8"]:
		return "8"
	elif num in ["nine", "9"]:
		return "9"
	elif num in ["zero", "0"]:
		return "0"
	return DEFAULT

# Apply both layers of phonetic mapping<commit_msg>small cleanups<commit_after>import speech_recognition as sr
import os
import time
import json
import logging, sys
import multiprocessing 
import pprint
import csv
import new_filter_function
import threading
import googleapiclient
from collections import Counter


LEVEL = logging.INFO
logging.basicConfig(stream=sys.stderr, level=LEVEL)
logging.getLogger('oauth2client.transport').setLevel(logging.ERROR)
logging.getLogger('googleapiclient.discovery').setLevel(logging.CRITICAL)  
logging.getLogger('oauth2client.client').setLevel(logging.ERROR)

pp = pprint.PrettyPrinter(indent=4)


#DEFAULT = "X" # all un-identified digits remain unknown
DEFAULT = "6" # all un-identified digits are mapped to "6"

apis = ["googleCloud", "wit", "bing", "ibm", "google", "sphinx"]

#Simple homophone mapping, taking any exact matches and returning the digit (layer one mapping)
def homophone(num):
	if num in ["one", "1", "won"]:
		return "1"
	elif num in ["two", "to", "too", "2"]:
		return "2"
	elif num in ["three", "3"]:
		return "3"
	elif num in ["four", "for", "4", "fore"]:
		return "4"
	elif num in ["five", "5"]:
		return "5"
	elif num in ["six", "6"]:
		return "6"
	elif num in ["seven", "7"]:
		return "7"
	elif num in ["eight", "ate", "8"]:
		return "8"
	elif num in ["nine", "9"]:
		return "9"
	elif num in ["zero", "0"]:
		return "0"
	return DEFAULT

# Apply both layers of phonetic mapping
{'fraction_non_alphanumeric': 0.14913544668587897, 'fraction_numerical': 0.018011527377521614, 'mean_word_length': 5.103296703296703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7597533', 'n_tokens_mistral': 984, 'n_tokens_neox': 939, 'n_words': 322}