<commit_before>from chainer import cuda
from chainer import function
from chainer import utils
import numpy


class ConnectionistTemporalClassification(function.Function):

    def __init__(self, blank_symbol):
        self.blank_symbol = blank_symbol
        self.epsilon = 1e-10

    '''
    Transtion in forword and backword algorithms is represented as matrix.
    See also
    https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
    '''
    def recurrence_relation(self, size, vtype):
        big_I = numpy.eye(size+2)
        rr = numpy.log((numpy.eye(size) + big_I[2:, 1:-1] +
                        big_I[2:, :-2] * (numpy.arange(size) % 2)))
        if vtype == numpy:
            return rr
        else:
            return cuda.to_gpu(rr)

    def label_to_path(self, labels):
        label_length = labels.shape[0]
        path = numpy.full((label_length * 2 + 1,),
                          self.blank_symbol, dtype=int)
        for i in range(label_length):
            path[i * 2 + 1] = labels[i]
        return path

    def log_dot(self, prob, rr):
        vtype = cuda.cupy.get_array_module(prob)
        res = vtype.zeros(prob.shape)
        rtrans = vtype.swapaxes(rr, 1, 0)
        for i in range(rtrans.shape[0]):
            res[i] = self.logsumexp(prob + rtrans[i])
        return res

    def logsumexp(self, a):
        vtype = cuda.cupy.get_array_module(a)
        vmax = vtype.amax(a)
        res = vtype.log(vtype.sum(vtype.exp(a - vmax)))
        return (res + vmax).astype(numpy.float32)

    # path probablity to label probability
    def label_probability(self, label_size, path, multiply):
        labels_prob = numpy.log(numpy.zeros(label_size))
        chars = set([c for c in path])
        for c in chars:
            pos = numpy.where(path == c)[0]
            labels_prob[c] = self.logsumexp(multiply[pos, ])
        return labels_prob

    def forward_cpu(self, inputs):
        t = inputs[0]
        yseq = numpy.log(inputs[1::])
        path = self.label_to_path(t)
        rr = self.recurrence_relation(path.shape[0],
                                      cuda.cupy.get_array_module(yseq))
        forward_prob_trans, backward_prob_trans \
            = self.calc_trans_cpu(path, yseq, rr)
        return utils.force_array(
            - self.logsumexp(forward_prob_trans[-1]
                             + backward_prob_trans[-1])).astype(numpy.float32),

    def calc_trans_cpu(self, path, yseq, rr):<commit_msg>fixes -inf when creating log value.<commit_after>from chainer import cuda
from chainer import function
from chainer import utils
import numpy


class ConnectionistTemporalClassification(function.Function):

    def __init__(self, blank_symbol):
        self.blank_symbol = blank_symbol
        self.epsilon = 1e-40

    '''
    Transtion in forword and backword algorithms is represented as matrix.
    See also
    https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
    '''
    def recurrence_relation(self, size, vtype):
        big_I = numpy.eye(size+2)
        rr = numpy.log((numpy.eye(size) + big_I[2:, 1:-1] +
                        big_I[2:, :-2] * (numpy.arange(size) % 2)) + self.epsilon)
        if vtype == numpy:
            return rr
        else:
            return cuda.to_gpu(rr)

    def label_to_path(self, labels):
        label_length = labels.shape[0]
        path = numpy.full((label_length * 2 + 1,),
                          self.blank_symbol, dtype=int)
        for i in range(label_length):
            path[i * 2 + 1] = labels[i]
        return path

    def log_dot(self, prob, rr):
        vtype = cuda.cupy.get_array_module(prob)
        res = vtype.zeros(prob.shape)
        rtrans = vtype.swapaxes(rr, 1, 0)
        for i in range(rtrans.shape[0]):
            res[i] = self.logsumexp(prob + rtrans[i])
        return res

    def logsumexp(self, a):
        vtype = cuda.cupy.get_array_module(a)
        vmax = vtype.amax(a)
        res = vtype.log(vtype.sum(vtype.exp(a - vmax)))
        return (res + vmax).astype(numpy.float32)

    # path probablity to label probability
    def label_probability(self, label_size, path, multiply):
        labels_prob = numpy.log(numpy.zeros(label_size) + self.epsilon)
        chars = set([c for c in path])
        for c in chars:
            pos = numpy.where(path == c)[0]
            labels_prob[c] = self.logsumexp(multiply[pos, ])
        return labels_prob

    def forward_cpu(self, inputs):
        t = inputs[0]
        yseq = numpy.log(inputs[1::])
        path = self.label_to_path(t)
        rr = self.recurrence_relation(path.shape[0],
                                      cuda.cupy.get_array_module(yseq))
        forward_prob_trans, backward_prob_trans \
            = self.calc_trans_cpu(path, yseq, rr)
        return utils.force_array(
            - self.logsumexp(forward_prob_trans[-1]
                             + backward_prob_trans[-1])).astype(numpy.float32),

    def calc_trans_cpu(self, path, yseq, rr):
{'fraction_non_alphanumeric': 0.11397934868943606, 'fraction_numerical': 0.014297061159650517, 'mean_word_length': 2.179924242424242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1171058', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1641, 'n_words': 342}