<commit_before>#!python
#coding= latin-1
# This script implements the Double Metaphone algorithm (c) 1998, 1999 by <NAME>
# it was translated to Python from the C source written by <NAME> (http://aspell.net/metaphone/)
# By <NAME> - January 12, 2007 who claims no rights to this work
# http://atomboy.isa-geek.com/plone/Members/acoil/programing/double-metaphone
# Tested with Python 2.4.3
# Updated Feb 14, 2007 - Found a typo in the 'gh' section
# Updated Dec 17, 2007 - Bugs fixed in 'S', 'Z', and 'J' sections. Thanks Chris Leong!
# Updated 2009-03-05 by <NAME> - Various bug fixes against the reference C++ implementation.

def dm(st) :
	"""dm(string) -> (string, string or None)
	returns the double metaphone codes for given string - always a tuple
	there are no checks done on the input string, but it should be a single	word or name."""
	vowels = ['A', 'E', 'I', 'O', 'U', 'Y']
	st = st.decode('ascii', 'ignore')
	st = st.upper() # st is short for string. I usually prefer descriptive over short, but this var is used a lot!
	is_slavo_germanic = (st.find('W') > -1 or st.find('K') > -1 or st.find('CZ') > -1 or st.find('WITZ') > -1)
	length = len(st)
	first = 2
	st = '-' * first + st + '------'  # so we can index beyond the begining and end of the input string
	last = first + length -1
	pos = first # pos is short for position
	pri = sec = '' # primary and secondary metaphone codes
	#skip these silent letters when at start of word
	if st[first:first+2] in ["GN", "KN", "PN", "WR", "PS"] :
		pos += 1
	# Initial 'X' is pronounced 'Z' e.g. 'Xavier'
	if st[first] == 'X' :
		pri = sec = 'S' #'Z' maps to 'S'
		pos += 1
	# main loop through chars in st
	while pos <= last :
		#print str(pos) + '\t' + st[pos]
		ch = st[pos] # ch is short for character
		# nxt (short for next characters in metaphone code) is set to  a tuple of the next characters in
		# the primary and secondary codes and how many characters to move forward in the string.
		# the secondary code letter is given only when it is different than the primary.
		# This is just a trick to make the code easier to write and read.
		nxt = (None, 1) # default action is to add nothing and move to next char
		if ch in vowels :
			nxt = (None, 1)
			if pos == first : # all init vowels now map to 'A'
				nxt = ('A', 1)
		elif ch == 'B' :
			#"-mb", e.g", "dumb", already skipped over... see 'M' below<commit_msg>Work better with Unicode data.<commit_after>#!python
#coding= latin-1
# This script implements the Double Metaphone algorithm (c) 1998, 1999 by <NAME>
# it was translated to Python from the C source written by <NAME> (http://aspell.net/metaphone/)
# By <NAME> - January 12, 2007 who claims no rights to this work
# http://atomboy.isa-geek.com/plone/Members/acoil/programing/double-metaphone
# Tested with Python 2.4.3
# Updated Feb 14, 2007 - Found a typo in the 'gh' section
# Updated Dec 17, 2007 - Bugs fixed in 'S', 'Z', and 'J' sections. Thanks <NAME>!
# Updated 2009-03-05 by <NAME> - Various bug fixes against the reference C++ implementation.

import unicodedata

def dm(st) :
	"""dm(string) -> (string, string or None)
	returns the double metaphone codes for given string - always a tuple
	there are no checks done on the input string, but it should be a single	word or name."""
	vowels = ['A', 'E', 'I', 'O', 'U', 'Y']
	st = ''.join((c for c in unicodedata.normalize('NFD', st) if unicodedata.category(c) != 'Mn'))
	st = st.upper() # st is short for string. I usually prefer descriptive over short, but this var is used a lot!
	is_slavo_germanic = (st.find('W') > -1 or st.find('K') > -1 or st.find('CZ') > -1 or st.find('WITZ') > -1)
	length = len(st)
	first = 2
	st = '-' * first + st + '------'  # so we can index beyond the begining and end of the input string
	last = first + length -1
	pos = first # pos is short for position
	pri = sec = '' # primary and secondary metaphone codes
	#skip these silent letters when at start of word
	if st[first:first+2] in ["GN", "KN", "PN", "WR", "PS"] :
		pos += 1
	# Initial 'X' is pronounced 'Z' e.g. 'Xavier'
	if st[first] == 'X' :
		pri = sec = 'S' #'Z' maps to 'S'
		pos += 1
	# main loop through chars in st
	while pos <= last :
		#print str(pos) + '\t' + st[pos]
		ch = st[pos] # ch is short for character
		# nxt (short for next characters in metaphone code) is set to  a tuple of the next characters in
		# the primary and secondary codes and how many characters to move forward in the string.
		# the secondary code letter is given only when it is different than the primary.
		# This is just a trick to make the code easier to write and read.
		nxt = (None, 1) # default action is to add nothing and move to next char
		if ch in vowels :
			nxt = (None, 1)
			if pos == first : # all init vowels now map to 'A'
				nxt = ('A', 1)
		elif ch == 'B' :
			#"-mb", e.g", "dumb", already skipped over... see 'M' below
{'fraction_non_alphanumeric': 0.13120494335736355, 'fraction_numerical': 0.02059732234809475, 'mean_word_length': 3.3279857397504458, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4348674', 'n_tokens_mistral': 1767, 'n_tokens_neox': 1615, 'n_words': 739}