<commit_before>'''
A module for representing the orthophonology of a language: 
the mapping from orthographic representations to IPA symbols.

Based on many ideas in cltk.phonology.utils by <NAME> <<EMAIL>>.
'''

from enum import IntEnum, auto
from copy import deepcopy
from functools import reduce
import re

__author__ = ["<NAME> <<EMAIL>>"]

# ------------------- Phonological Features -------------------

# The list of features and their values are from the IPA charts.
# Features for non-pulmonic consonants (e.g. clicks, implosives) are not yet provided.

class PhonologicalFeature(IntEnum):
	def __sub__(self, other):
		return phoneme(self) - other

	def __rshift__(self, other):
		return phoneme(self) >> other

	def __le__(self, other):
		return phoneme(self) <= other

	def __ge__(self, other):
		return phoneme(self) >= other

	def matches(self, other):
		return phoneme(self).matches(other)

class Consonantal(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Voiced(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Aspirated(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Geminate(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Roundedness(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Length(PhonologicalFeature):
    short = auto()
    long = auto()
    overlong = auto()

# order for Height, Backness, and Manner is important
# the feature values must be ordered by *increasing sonority*
class Height(PhonologicalFeature):
    close = auto()
    near_close = auto()
    close_mid = auto()
    mid = auto()
    open_mid = auto()
    near_open = auto()
    open = auto()

class Backness(PhonologicalFeature):
    front = auto()
    central = auto()
    back = auto()

class Manner(PhonologicalFeature):
	stop = auto()
	fricative = auto()
	affricate = auto()
	nasal = auto()
	lateral = auto()
	trill = auto()
	spirant = auto()
	approximant = auto()

class Place(PhonologicalFeature):
    bilabial = auto()
    labio_dental = auto()
    dental = auto()
    alveolar = auto()
    post_alveolar = auto()
    retroflex = auto()
    palatal = auto()
    velar = auto()
    uvular = auto()
    glottal = auto()


# ------------------- Phonemes -------------------

class AbstractPhoneme:
	'''
	An abstract phoneme is just a bundle of phonological features.
	'''

	def __init__(self, features = {}, ipa = None):
		# ensure unique features
		if len(set(features.keys())) != len(features.keys()):
			raise ValueError('non-unique features')

		# ensure feature values correctly match their types
		# this is a barbaric bit of type checking that the language should provide
		for feature_name, feature_value in features.items():
			if not issubclass(feature_name, PhonologicalFeature):
				raise TypeError(str(feature_name) + ' is not a phonological feature')
			if type(feature_value) != feature_name:
				raise TypeError(str(feature_value) + ' is not a ' + str(feature_name))

		self.features = features
		self.ipa = ipa

	def is_vowel(self):
		return self[Consonantal] == Consonantal.neg

	def _check_disjunctive_features(self, feature_values):
		return reduce(lambda a, b: a or b, [self[type(f)] == f for f in feature_values])

	def merge(self, other):
		phoneme = deepcopy(self)

		# special case for list of phonemes
		if type(other) == list and len(other) > 0 and issubclass(type(other[0]), AbstractPhoneme):
			return other

		if issubclass(type(other), AbstractPhoneme):
			feature_values = other.features.values()
		elif type(other) != list and type(other) != tuple:
			feature_values = [other]
		else:
			feature_values = other

		for f in feature_values:
			if type(f) == list:
				for inner_f in f:
					phoneme[type(inner_f)] = inner_f
			elif issubclass(type(f), AbstractPhoneme):
				phoneme = phoneme << f
			else:
				phoneme[type(f)] = f

		if issubclass(type(other), AbstractPhoneme) and other.ipa is not None:
			phoneme.ipa = other.ipa

		return phoneme

	def is_equal(self, other):
		return self.features == other.features

	def matches(self, other):
		if other is None:
			return False
		if type(other) == list or issubclass(type(other), PhonologicalFeature):
			other = phoneme(other)
		return other.features.items() >= self.features.items()

	def __getitem__(self, feature_name):
		'''
		Use dict-type syntax for accessing the values of features.
		'''
		if not issubclass(feature_name, PhonologicalFeature):
			raise TypeError(str(feature_name) + ' is not a phonological feature')
		return self.features.get(feature_name, None)

	def __setitem__(self, feature_name, feature_value):
		if not issubclass(feature_name, PhonologicalFeature):
			raise TypeError(str(feature_name) + ' is not a phonological feature')
		if type(feature_value) != feature_name:
			raise TypeError(str(feature_value) + ' is not a ' + str(feature_name))
		self.features[feature_name] = feature_value
		return self

	def __str__(self):
		return  ('IPA:{0} '.format(self.ipa) if self.ipa is not None else '') + \
		' '.join([str(v) for v in self.features.values()])

	__repr__ = __str__

	def __eq__(self, other):
		return self.is_equal(other)

	def __le__ (self, other):
		return self.matches(other)

	def __ge__ (self, other):
		return other.matches(self)

	def __lt__ (self, other):
		return other.is_more_sonorous(self)

	def __gt__ (self, other):
		return self.is_more_sonorous(other)

	def __rshift__(self, other):
		return PhonologicalRule(
			condition = lambda _, target, __: self <= target,
			action = lambda target : target << other)

	def __lshift__(self, other):
		return self.merge(other)

	def __sub__(self, other):
		other = phoneme(other) if not issubclass(type(other), AbstractPhoneme) else other
		return lambda before, _, after : self <= before and other <= after

def phoneme(*feature_values):
	phoneme = AbstractPhoneme({})
	phoneme = phoneme << feature_values
	return phoneme

class AlwaysMatchingPhoneme(AbstractPhoneme):
	def matches(self, other):
		return True

class NeverMatchingPhoneme(AbstractPhoneme):
	def matches(self, other):
		return False
	
	def is_equal(self, other):
		return self is other

ANY = AlwaysMatchingPhoneme()
W = NeverMatchingPhoneme(ipa = '#')
S = NeverMatchingPhoneme(ipa = '$')


class PositionedPhoneme(AbstractPhoneme):
	def __init__(self, from_phoneme, 
		word_initial = False, word_final = False, 
		syllable_initial = False, syllable_final = False,
		env_start = False, env_end = False):

		self.word_initial = word_initial
		self.word_final = word_final
		self.syllable_initial = syllable_initial
		self.syllable_final = syllable_final
		self.env_start = env_start
		self.env_end = env_end

		AbstractPhoneme.__init__(self, from_phoneme.features)
		


class Consonant(AbstractPhoneme):
	'''
	Based on cltk.phonology.utils by @clemsciences.
	A consonant is a phoneme that is specified for the features listed in the IPA chart for consonants:
	Place, Manner, Voicing.  These may be read directly off the IPA chart, which also gives the IPA symbol.
	The Consonantal feature is set to positive, and the aspirated is defaulted to negative.
	See http://www.ipachart.com/
	'''

	def __init__(self, place, manner, voiced, ipa, geminate = Geminate.neg, aspirated = Aspirated.neg):
		assert place is not None
		assert manner is not None
		assert voiced is not None
		assert geminate is not None
		assert ipa is not None

		AbstractPhoneme.__init__(self, {
			Consonantal : Consonantal.pos,
			Place       : place,
			Manner      : manner,
			Voiced      : voiced,
			Aspirated   : aspirated,
			Geminate    : geminate}, 
			ipa)

	def is_more_sonorous(self, other):
		'''<commit_msg>experiment with top-down rule system<commit_after>'''
A module for representing the orthophonology of a language: 
the mapping from orthographic representations to IPA symbols.

Based on many ideas in cltk.phonology.utils by <NAME> <<EMAIL>>.
'''

from enum import IntEnum, auto
from copy import deepcopy
from functools import reduce
import re

__author__ = ["<NAME> <<EMAIL>>"]

# ------------------- Phonological Features -------------------

# The list of features and their values are from the IPA charts.
# Features for non-pulmonic consonants (e.g. clicks, implosives) are not yet provided.

class PhonologicalFeature(IntEnum):
	def __sub__(self, other):
		return phoneme(self) - other

	def __rshift__(self, other):
		return phoneme(self) >> other

	def __le__(self, other):
		return phoneme(self) <= other

	def __ge__(self, other):
		return phoneme(self) >= other

	def matches(self, other):
		return phoneme(self).matches(other)

class Consonantal(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Voiced(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Aspirated(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Geminate(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Roundedness(PhonologicalFeature):
	neg = auto()
	pos = auto()

class Length(PhonologicalFeature):
    short = auto()
    long = auto()
    overlong = auto()

# order for Height, Backness, and Manner is important
# the feature values must be ordered by *increasing sonority*
class Height(PhonologicalFeature):
    close = auto()
    near_close = auto()
    close_mid = auto()
    mid = auto()
    open_mid = auto()
    near_open = auto()
    open = auto()

class Backness(PhonologicalFeature):
    front = auto()
    central = auto()
    back = auto()

class Manner(PhonologicalFeature):
	stop = auto()
	fricative = auto()
	affricate = auto()
	nasal = auto()
	lateral = auto()
	trill = auto()
	spirant = auto()
	approximant = auto()

class Place(PhonologicalFeature):
    bilabial = auto()
    labio_dental = auto()
    dental = auto()
    alveolar = auto()
    post_alveolar = auto()
    retroflex = auto()
    palatal = auto()
    velar = auto()
    uvular = auto()
    glottal = auto()


# ------------------- Phonemes -------------------

class AbstractPhoneme:
	'''
	An abstract phoneme is just a bundle of phonological features.
	'''

	def __init__(self, features = {}, ipa = None):
		# ensure unique features
		if len(set(features.keys())) != len(features.keys()):
			raise ValueError('non-unique features')

		# ensure feature values correctly match their types
		# this is a barbaric bit of type checking that the language should provide
		for feature_name, feature_value in features.items():
			if not issubclass(feature_name, PhonologicalFeature):
				raise TypeError(str(feature_name) + ' is not a phonological feature')
			if type(feature_value) != feature_name:
				raise TypeError(str(feature_value) + ' is not a ' + str(feature_name))

		self.features = features
		self.ipa = ipa

	def is_vowel(self):
		return self[Consonantal] == Consonantal.neg

	def _check_disjunctive_features(self, feature_values):
		return reduce(lambda a, b: a or b, [self[type(f)] == f for f in feature_values])

	def merge(self, other):
		phoneme = deepcopy(self)

		# special case for list of phonemes
		if type(other) == list and len(other) > 0 and issubclass(type(other[0]), AbstractPhoneme):
			return other

		if issubclass(type(other), AbstractPhoneme):
			feature_values = other.features.values()
		elif type(other) != list and type(other) != tuple:
			feature_values = [other]
		else:
			feature_values = other

		for f in feature_values:
			if type(f) == list:
				for inner_f in f:
					phoneme[type(inner_f)] = inner_f
			elif issubclass(type(f), AbstractPhoneme):
				phoneme = phoneme << f
			else:
				phoneme[type(f)] = f

		if issubclass(type(other), AbstractPhoneme) and other.ipa is not None:
			phoneme.ipa = other.ipa

		return phoneme

	def is_equal(self, other):
		return self.features == other.features

	def matches(self, other):
		if other is None:
			return False
		if type(other) == list or issubclass(type(other), PhonologicalFeature):
			other = phoneme(other)
		return other.features.items() >= self.features.items()

	def __getitem__(self, feature_name):
		'''
		Use dict-type syntax for accessing the values of features.
		'''
		if not issubclass(feature_name, PhonologicalFeature):
			raise TypeError(str(feature_name) + ' is not a phonological feature')
		return self.features.get(feature_name, None)

	def __setitem__(self, feature_name, feature_value):
		if not issubclass(feature_name, PhonologicalFeature):
			raise TypeError(str(feature_name) + ' is not a phonological feature')
		if type(feature_value) != feature_name:
			raise TypeError(str(feature_value) + ' is not a ' + str(feature_name))
		self.features[feature_name] = feature_value
		return self

	def __str__(self):
		return  ('IPA:{0} '.format(self.ipa) if self.ipa is not None else '') + \
		' '.join([str(v) for v in self.features.values()])

	__repr__ = __str__

	def __eq__(self, other):
		return self.is_equal(other)

	def __le__ (self, other):
		return self.matches(other)

	def __ge__ (self, other):
		return other.matches(self)

	def __lt__ (self, other):
		return other.is_more_sonorous(self)

	def __gt__ (self, other):
		return self.is_more_sonorous(other)

	def __rshift__(self, other):
		return PhonologicalRule(
			condition = lambda _, target, __: self <= target,
			action = lambda target : target << other)

	def __lshift__(self, other):
		return self.merge(other)

	def __sub__(self, other):
		other = phoneme(other) if not issubclass(type(other), AbstractPhoneme) else other
		return lambda before, _, after : self <= before and other <= after

def phoneme(*feature_values):
	phoneme = AbstractPhoneme({})
	phoneme = phoneme << feature_values
	return phoneme

class AlwaysMatchingPhoneme(AbstractPhoneme):
	def matches(self, other):
		return True

class NeverMatchingPhoneme(AbstractPhoneme):
	def matches(self, other):
		return False
	
	def is_equal(self, other):
		return self is other

ANY = AlwaysMatchingPhoneme()
W = NeverMatchingPhoneme(ipa = '#')
S = NeverMatchingPhoneme(ipa = '$')

def is_syllable_initial(phonemes, pos):
	if pos == len(phonemes) - 1:
		return False
	return pos == 0 or \
	(phonemes[pos - 1].is_more_sonorous(phonemes[pos]) and not phonemes[pos].is_more_sonorous(phonemes[pos + 1]))

def is_syllable_final(phonemes, pos):
	if pos == 0:
		return False
	return pos == len(phonemes) -1  or \
	(phonemes[pos + 1].is_more_sonorous(phonemes[pos]) and not phonemes[pos].is_more_sonorous(phonemes[pos - 1]))


class Consonant(AbstractPhoneme):
	'''
	Based on cltk.phonology.utils by @clemsciences.
	A consonant is a phoneme that is specified for the features listed in the IPA chart for consonants:
	Place, Manner, Voicing.  These may be read directly off the IPA chart, which also gives the IPA symbol.
	The Consonantal feature is set to positive, and the aspirated is defaulted to negative.
	See http://www.ipachart.com/
	'''

	def __init__(self, place, manner, voiced, ipa, geminate = Geminate.neg, aspirated = Aspirated.neg):
		assert place is not None
		assert manner is not None
		assert voiced is not None
		assert geminate is not None
		assert ipa is not None

		AbstractPhoneme.__init__(self, {
			Consonantal : Consonantal.pos,
			Place       : place,
			Manner      : manner,
			Voiced      : voiced,
			Aspirated   : aspirated,
			Geminate    : geminate}, 
			ipa)

	def is_more_sonorous(self, other):
		'''
{'fraction_non_alphanumeric': 0.11617462518988178, 'fraction_numerical': 0.0009246417013407304, 'mean_word_length': 3.992416749093307, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985489', 'n_tokens_mistral': 5315, 'n_tokens_neox': 4871, 'n_words': 1479}