<commit_before>##########################################################################
#Copyright 2015 <NAME>                                              #
#                                                                        #
#Licensed under the Apache License, Version 2.0 (the "License");         #
#you may not use this file except in compliance with the License.        #
#You may obtain a copy of the License at                                 #
#                                                                        #
#http://www.apache.org/licenses/LICENSE-2.0                              #
#                                                                        #
#Unless required by applicable law or agreed to in writing, software     #
#distributed under the License is distributed on an "AS IS" BASIS,       #
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.#
#See the License for the specific language governing permissions and     #
#limitations under the License.                                          #
##########################################################################

import utterance
from error_messages import SiReError

#Methods for manipulating utterances in non-standard ways. Most of these are considered dangerous and unsafe.

#Try to split words in the utterance which may be contraction which parsing expands.
#This is potentially unsafe so outputs warnings when doing it.
#This list is likely incomplete, but it is also likely not all issues can be resolved.
def try_split_words(utt):
  l = len(utt.words)
  #We should not try this if we have not gotten the word.id from txt.
  if utt.txtloaded != True:
    raise SiReError("Cannot split words if word ids not loaded from txt.")
  for word in utt.words:
    #End of word 's
    if word.id[-2:] == "'s":
      split_word(word, -2)
    #Contracted are's (e.g. we're)
    elif word.id[-3:] == "'re":
      split_word(word, -3)
    #Contracted not's (e.g. don't)
    elif word.id[-3:] == "n't":
      split_word(word, -2)
    #Contracted will's (e.g. it'll)
    elif word.id[-3:] == "'ll":
      split_word(word, -3)
    #Contracted have's (e.g. I've)
    elif word.id[-3:] == "'ve":
      split_word(word, -3)
    #Contracted I am
    elif word.id == "i'm":
      split_word(word, -2)
    #Contracted would or had (e.g. she'd)
    elif word.id[-2:] == "'d":
      split_word(word, -2)
    #Contracted going to
    elif word.id == "gonna":
      split_word(word, -2)
    #Contracted can not
    elif word.id == "cannot":
      split_word(word, -3)
    #Contracted want to
    elif word.id == "wanna":
      split_word(word, -2)
  if l == len(utt.words):
    print "Warning: Nothing to split in word."

#Splits a word into two, the new being its last syllable.
#Split_pos is the character pos in the word from which to split.
def split_word(word, split_pos):
  utt = word.parent_utt
  split_more_than_one_phoneme = False
  #We usually only want to change one phoneme and this list checks for those.
  if word.syllables[-1].num_phonemes() != 1:
    s = word.syllables[-1]
    #This should also update the word itself with the new syll info.
    #End of word 's
    if word.id[-2:] == "'s":
      split_syll(s, ["s", "z"])
    #Contracted are's (e.g. we're)
    elif word.id[-3:] == "'re":
      #'r' is a bit... meh. In e.g. "you're" pronounced "jU@r" we kinda want to add a phony
      #before "r".
      #But this is not supported atm.
      split_syll(s, ["I@", "U@", "E@", "@", "r"], ["I@", "U@", "E@", "O"])
    #Contracted not's (e.g. don't)
    elif word.id[-3:] == "n't":
      split_syll(s, ["n", "G", "t"])
    #Contracted will's (e.g. it'll)
    elif word.id[-3:] == "'ll":
      split_syll(s, ["lw", "l"])
    #Contracted have's (e.g. I've)
    elif word.id[-3:] == "'ve":
      split_syll(s, ["f", "v"])
    #Contracted I am
    elif word.id[-3:] == "i'm":
      split_syll(s, ["m"])
    #Contracted would or had (e.g. she'd)
    elif word.id[-2:] == "'d":
      split_syll(s, ["d", "G"], ["u"])
    #Contracted going to
    elif word.id == "gonna":
      if s.id in ["nu", "n@"]:
        split_more_than_one_phoneme = True
    #Contracted can not
    elif word.id == "cannot":
      #If the syll is "nQG" or "nQt" we're good and can split
      if s.id in ["nQG", "nQt"]:
        split_more_than_one_phoneme = True
    elif word.id == "wanna":
      #If the syll is "n@" we're good and can split
      if s.id == "n@":
        split_more_than_one_phoneme = True
  #If there is only one syllable with one word we have to add a phony syll.
  elif len(word.syllables) < 2:
    s = word.syllables[-1]
    #Contracted would or had (e.g. I'd)
    if word.id[-2:] == "'d":
      split_syll(s, ["aI"], ["aI"])
  
  if word.syllables[-1].num_phonemes() > 1 and split_more_than_one_phoneme != True:
    raise SiReError("Cannot split a word {0} with final syllable {1} with more than one phoneme ({2}) as this has not been explicitly allowed!".format(word.id, word.syllables[-1].id, word.syllables[-1].num_phonemes()))
  w1 = utterance.Word()
  w1.id = word.id[:split_pos]
  w2 = utterance.Word()
  w2.id = word.id[split_pos:]
  print "Warning: Splitting word ({0}) into two ({1} and {2}). Is this correct?".format(word.id, w1.id, w2.id)
  #Start time
  w1.start = word.start_time()
  w2.start = word.syllables[-1].start_time()
  #End time
  w1.end = word.syllables[-2].end_time()
  w2.end = word.end_time()
  #Parent utt
  w1.parent_utt = utt
  w2.parent_utt = utt
  #Pos in utt
  #Slice out the original word
  w_p_u = word.pos_in_utt()
  utt.words = utt.words[:w_p_u] + [w1, w2] + utt.words[w_p_u + 1:]
  #Fix syllables
  w1.syllables = word.syllables[:-1]
  w2.syllables = [word.syllables[-1]]
  for s in w1.syllables:
    s.parent_word = w1
  w2.syllables[0].parent_word = w2
  #Fix phonemes
  w1.phonemes = word.phonemes[:-len(word.syllables[-1].phonemes)]
  w2.phonemes = word.phonemes[-len(word.syllables[-1].phonemes):]
  for p in w1.phonemes:
    p.parent_word = w1
  for p in w2.phonemes:
    p.parent_word = w2
  
  #Delete the original word. If all has gone well this should be fine.
  del word

#Splits a syllable on its last phoneme.
def split_syll(syll, acceptable_phoneme_set, word_spanning_phonemes=[]):
  #Makes life a bit easier
  utt = syll.parent_utt
  phoneme_features = utt.phoneme_features
  
  #A special case for phonemes which may have ended up spanning across what would normally
  #be two words or if all phonemes related to the "2nd" word has been deleted.
  #E.g. I@ in we're (w I@) or u in who'd (h u).
  #In this case we add a new "phony" syllable with no duration. So it affects contexts
  #but does not take any frames.<commit_msg>Small bug fix related to slplitting words, particularly I'd.<commit_after>##########################################################################
#Copyright 2015 <NAME>                                              #
#                                                                        #
#Licensed under the Apache License, Version 2.0 (the "License");         #
#you may not use this file except in compliance with the License.        #
#You may obtain a copy of the License at                                 #
#                                                                        #
#http://www.apache.org/licenses/LICENSE-2.0                              #
#                                                                        #
#Unless required by applicable law or agreed to in writing, software     #
#distributed under the License is distributed on an "AS IS" BASIS,       #
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.#
#See the License for the specific language governing permissions and     #
#limitations under the License.                                          #
##########################################################################

import utterance
from error_messages import SiReError

#Methods for manipulating utterances in non-standard ways. Most of these are considered dangerous and unsafe.

#Try to split words in the utterance which may be contraction which parsing expands.
#This is potentially unsafe so outputs warnings when doing it.
#This list is likely incomplete, but it is also likely not all issues can be resolved.
def try_split_words(utt):
  l = len(utt.words)
  #We should not try this if we have not gotten the word.id from txt.
  if utt.txtloaded != True:
    raise SiReError("Cannot split words if word ids not loaded from txt.")
  for word in utt.words:
    #End of word 's
    if word.id[-2:] == "'s":
      split_word(word, -2)
    #Contracted are's (e.g. we're)
    elif word.id[-3:] == "'re":
      split_word(word, -3)
    #Contracted not's (e.g. don't)
    elif word.id[-3:] == "n't":
      split_word(word, -2)
    #Contracted will's (e.g. it'll)
    elif word.id[-3:] == "'ll":
      split_word(word, -3)
    #Contracted have's (e.g. I've)
    elif word.id[-3:] == "'ve":
      split_word(word, -3)
    #Contracted I am
    elif word.id == "i'm":
      split_word(word, -2)
    #Contracted would or had (e.g. she'd)
    elif word.id[-2:] == "'d":
      split_word(word, -2)
    #Contracted going to
    elif word.id == "gonna":
      split_word(word, -2)
    #Contracted can not
    elif word.id == "cannot":
      split_word(word, -3)
    #Contracted want to
    elif word.id == "wanna":
      split_word(word, -2)
  if l == len(utt.words):
    print "Warning: Nothing to split in word."

#Splits a word into two, the new being its last syllable.
#Split_pos is the character pos in the word from which to split.
def split_word(word, split_pos):
  utt = word.parent_utt
  split_more_than_one_phoneme = False
  #We usually only want to change one phoneme and this list checks for those.
  if word.syllables[-1].num_phonemes() != 1:
    s = word.syllables[-1]
    #This should also update the word itself with the new syll info.
    #End of word 's
    if word.id[-2:] == "'s":
      split_syll(s, ["s", "z"])
    #Contracted are's (e.g. we're)
    elif word.id[-3:] == "'re":
      #'r' is a bit... meh. In e.g. "you're" pronounced "jU@r" we kinda want to add a phony
      #before "r".
      #But this is not supported atm.
      split_syll(s, ["I@", "U@", "E@", "@", "r"], ["I@", "U@", "E@", "O"])
    #Contracted not's (e.g. don't)
    elif word.id[-3:] == "n't":
      split_syll(s, ["n", "G", "t"])
    #Contracted will's (e.g. it'll)
    elif word.id[-3:] == "'ll":
      split_syll(s, ["lw", "l"])
    #Contracted have's (e.g. I've)
    elif word.id[-3:] == "'ve":
      split_syll(s, ["f", "v"])
    #Contracted I am
    elif word.id[-3:] == "i'm":
      split_syll(s, ["m"])
    #Contracted would or had (e.g. she'd)
    elif word.id[-2:] == "'d":
      split_syll(s, ["d", "G"], ["u"])
    #Contracted going to
    elif word.id == "gonna":
      if s.id in ["nu", "n@"]:
        split_more_than_one_phoneme = True
    #Contracted can not
    elif word.id == "cannot":
      #If the syll is "nQG" or "nQt" we're good and can split
      if s.id in ["nQG", "nQt"]:
        split_more_than_one_phoneme = True
    elif word.id == "wanna":
      #If the syll is "n@" we're good and can split
      if s.id == "n@":
        split_more_than_one_phoneme = True
  #If there is only one syllable with one word we have to add a phony syll.
  elif len(word.syllables) < 2:
    s = word.syllables[-1]
    #Contracted would or had (e.g. I'd)
    if word.id[-2:] == "'d":
      split_syll(s, ["aI"], ["aI"])
  
  if word.syllables[-1].num_phonemes() > 1 and split_more_than_one_phoneme != True:
    raise SiReError("Cannot split a word {0} with final syllable {1} with more than one phoneme ({2}) as this has not been explicitly allowed!".format(word.id, word.syllables[-1].id, word.syllables[-1].num_phonemes()))
  w1 = utterance.Word()
  w1.id = word.id[:split_pos]
  w2 = utterance.Word()
  w2.id = word.id[split_pos:]
  print "Warning: Splitting word ({0}) into two ({1} and {2}). Is this correct?".format(word.id, w1.id, w2.id)
  #Start time
  w1.start = word.start_time()
  w2.start = word.syllables[-1].start_time()
  #End time
  w1.end = word.syllables[-2].end_time()
  w2.end = word.end_time()
  #Parent utt
  w1.parent_utt = utt
  w2.parent_utt = utt
  #Pos in utt
  #Slice out the original word
  w_p_u = word.pos_in_utt()
  utt.words = utt.words[:w_p_u] + [w1, w2] + utt.words[w_p_u + 1:]
  #Fix syllables and phonemes
  w1.syllables = word.syllables[:-1]
  w2.syllables = [word.syllables[-1]]
  w1.phonemes = []
  w2.phonemes = []
  for s in w1.syllables:
    s.parent_word = w1
    for p in s.phonemes:
      w1.phonemes.append(p)
      p.parent_word = w1
  w2.syllables[0].parent_word = w2
  for p in w2.syllables[0].phonemes:
    w2.phonemes.append(p)
    p.parent_word = w2
#  #Fix phonemes
#  w1.phonemes = word.phonemes[:-len(word.syllables[-1].phonemes)]
#  w2.phonemes = word.phonemes[-len(word.syllables[-1].phonemes):]
#  for p in w1.phonemes:
#    p.parent_word = w1
#  for p in w2.phonemes:
#    p.parent_word = w2
  
  #Delete the original word. If all has gone well this should be fine.
  del word

#Splits a syllable on its last phoneme.
def split_syll(syll, acceptable_phoneme_set, word_spanning_phonemes=[]):
  #Makes life a bit easier
  utt = syll.parent_utt
  phoneme_features = utt.phoneme_features
  
  #A special case for phonemes which may have ended up spanning across what would normally
  #be two words or if all phonemes related to the "2nd" word has been deleted.
  #E.g. I@ in we're (w I@) or u in who'd (h u).
  #Or a deleted stop in I'd (aI).
  #In this case we add a new "phony" syllable with no duration. So it affects contexts
  #but does not take any frames.
{'fraction_non_alphanumeric': 0.15401475852999197, 'fraction_numerical': 0.012566669102067655, 'mean_word_length': 2.5160544567171845, 'pattern_counts': {'":': 44, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '634941', 'n_tokens_mistral': 4871, 'n_tokens_neox': 4624, 'n_words': 1542}