<commit_before># coding=utf-8
import re

def load_vocabulary(words_file):
    return set([X.split(' ')[0] for X in open(words_file).read().split('\n')])

def kaldi_normalize(txt, vocab):
    # lowercase
    norm = txt.lower()
    # Turn fancy apostrophes into simpler apostrophes
    norm = norm.replace("’", "'")
    # preserve in-vocab hyphenated phrases
    if norm in vocab:
        return [norm]
    # turn hyphens into spaces
    norm = norm.replace('-', ' ')
    # remove all punctuation
    norm = re.sub(r'[^a-z0-9\s\']', ' ', norm)
    seq = norm.split()
    # filter out empty words
    seq = [x.strip() for x in seq if len(x.strip())>0]
    # replace [oov] words
    seq = [x if x in vocab else '[oov]' for x in seq]

    return seq

class MetaSentence:
    """Maintain two parallel representations of a sentence: one for
    Kaldi's benefit, and the other in human-legible form.
    """

    def __init__(self, sentence, vocab):
        self.raw_sentence = sentence
        self.vocab = vocab

        self._gen_kaldi_seq(sentence)

    def _gen_kaldi_seq(self, sentence):
        self._seq = []
        for m in re.finditer(r'[^ \n]+', sentence):
            start, end = m.span()
            word = m.group()
            if len(word.strip()) == 0:
                continue
            token = kaldi_normalize(word, self.vocab)
            self._seq.append({
                "start": start,
                "end": end,
                "token": token,
            })

    def get_kaldi_sequence(self):
        return reduce(lambda acc,y: acc+y["token"], self._seq, [])

    def get_matched_kaldi_sequence(self):
        return ['-'.join(X["token"]) for X in self._seq]

    def get_display_sequence(self):
        display_sequence = []
        for x in self._seq:
            start, end = x["start"], x["end"]
            word = self.raw_sentence[start:end]
            display_sequence.append(word)
        return display_sequence

    def get_text_offsets(self):
        return [(x["start"], x["end"]) for x in self._seq]

    def align_from_kaldi(self, kaldi_alignment):
        """Return the display sentence, based on the alignment of the kaldi
        sentence
        """

        if len(self._seq) == 0:
            print 'no sequence?'
            return []
        
        display_timing = []
        cur_s_idx = 0
        cur_display_wd = {"word": self._seq[0]["display"]}
        
        started = False
        
        for wd in kaldi_alignment:
            s = self._seq[cur_s_idx]
            
            if len(s["kaldi"]) == 0:
                print 'k0', s
                cur_s_idx += 1
                if cur_s_idx == len(self._seq):
                    # Done!
                    cur_display_wd["word"] += "."
                    break
                cur_display_wd = {"word": self._seq[cur_s_idx]["display"]}
                s = self._seq[cur_s_idx]
                
            if s["kaldi"][0] == wd["word"]:
                # This is the first word in the chunk
                cur_display_wd["start"] = wd["start"]
                started = True
            if started and s["kaldi"][-1] == wd["word"]:
                # This is the last word in the chunk
                end = wd["start"] + wd["duration"]
                cur_display_wd["duration"] = end - cur_display_wd["start"]
                cur_s_idx += 1

                display_timing.append(cur_display_wd)
                started = False
                if cur_s_idx == len(self._seq):
                    # Done!
                    cur_display_wd["word"] += "."
                    break

                cur_display_wd = {"word": self._seq[cur_s_idx]["display"]}
            else:
                print 'what is this case?', 's', s, 'wd', wd, 'started', started

                #import pdb; pdb.set_trace()

        print 'AFK - DISPLAY TIMING', display_timing
        return display_timing<commit_msg>remove unused align_from_kaldi function<commit_after># coding=utf-8
import re

def load_vocabulary(words_file):
    return set([X.split(' ')[0] for X in open(words_file).read().split('\n')])

def kaldi_normalize(txt, vocab):
    # lowercase
    norm = txt.lower()
    # Turn fancy apostrophes into simpler apostrophes
    norm = norm.replace("’", "'")
    # preserve in-vocab hyphenated phrases
    if norm in vocab:
        return [norm]
    # turn hyphens into spaces
    norm = norm.replace('-', ' ')
    # remove all punctuation
    norm = re.sub(r'[^a-z0-9\s\']', ' ', norm)
    seq = norm.split()
    # filter out empty words
    seq = [x.strip() for x in seq if len(x.strip())>0]
    # replace [oov] words
    seq = [x if x in vocab else '[oov]' for x in seq]

    return seq

class MetaSentence:
    """Maintain two parallel representations of a sentence: one for
    Kaldi's benefit, and the other in human-legible form.
    """

    def __init__(self, sentence, vocab):
        self.raw_sentence = sentence
        self.vocab = vocab

        self._gen_kaldi_seq(sentence)

    def _gen_kaldi_seq(self, sentence):
        self._seq = []
        for m in re.finditer(r'[^ \n]+', sentence):
            start, end = m.span()
            word = m.group()
            if len(word.strip()) == 0:
                continue
            token = kaldi_normalize(word, self.vocab)
            self._seq.append({
                "start": start,
                "end": end,
                "token": token,
            })

    def get_kaldi_sequence(self):
        return reduce(lambda acc,y: acc+y["token"], self._seq, [])

    def get_matched_kaldi_sequence(self):
        return ['-'.join(X["token"]) for X in self._seq]

    def get_display_sequence(self):
        display_sequence = []
        for x in self._seq:
            start, end = x["start"], x["end"]
            word = self.raw_sentence[start:end]
            display_sequence.append(word)
        return display_sequence

    def get_text_offsets(self):
        return [(x["start"], x["end"]) for x in self._seq]
{'fraction_non_alphanumeric': 0.1274427927175547, 'fraction_numerical': 0.003507599799565726, 'mean_word_length': 1.8623326959847035, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3356835', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1820, 'n_words': 481}