<commit_before>from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals
from builtins import zip
from builtins import range
from builtins import object

import unicodedata
import numpy as np

from scipy.ndimage import measurements
from scipy.special import expit

initial_range = 0.1


class Codec(object):
    """Translate between integer codes and characters."""
    def init(self, charset):
        charset = sorted(list(set(charset)))
        self.code2char = {}
        self.char2code = {}
        for code,char in enumerate(charset):
            self.code2char[code] = char
            self.char2code[char] = code
        return self
    def size(self):
        """The total number of codes (use this for the number of output
        classes when training a classifier."""
        return len(list(self.code2char.keys()))
    def encode(self, s):
        "Encode the string `s` into a code sequence."
        tab = self.char2code
        dflt = self.char2code["~"]
        return [self.char2code.get(c,dflt) for c in s]
    def decode(self, l):
        "Decode a code sequence into a string."
        s = [self.code2char.get(c,"~") for c in l]
        return s


def normalize_nfkc(s):
    return unicodedata.normalize('NFKC',s)


def prepare_line(line, pad=16):
    """Prepare a line for recognition; this inverts it, transposes
    it, and pads it."""
    line = line * 1.0/np.amax(line)
    line = np.amax(line)-line
    line = line.T
    if pad>0:
        w = line.shape[1]
        line = np.vstack([np.zeros((pad,w)),line,np.zeros((pad,w))])
    return line


def randu(*shape):
    # ATTENTION: whether you use randu or randn can make a difference.
    """Generate uniformly random values in the range (-1,1).
    This can usually be used as a drop-in replacement for `randn`
    resulting in a different distribution."""
    return 2*np.random.rand(*shape)-1


def sigmoid(x):
    """
    Compute the sigmoid function. We don't bother with clipping the input
    value because IEEE floating point behaves reasonably with this function
    even for infinities. 
    
    Further we use scipy's expit function which is ~50% faster for decently
    sized arrays.
    """
    return expit(x)

# These are the nonlinearities used by the LSTM network.
# We don't bother parameterizing them here

def ffunc(x):
    "Nonlinearity used for gates."
    return 1.0/(1.0+np.exp(-x))
def gfunc(x):
    "Nonlinearity used for input to state."
    return np.tanh(x)
# ATTENTION: try linear for hfunc
def hfunc(x):
    "Nonlinearity used for output."
    return np.tanh(x)

################################################################
# LSTM classification with forward/backward alignment ("CTC")
################################################################

def translate_back(outputs, threshold=0.5, pos=0):
    """Translate back. Thresholds on class 0, then assigns
    the maximum class to each region."""
    labels, n = measurements.label(outputs[:,0] < threshold)
    mask = np.tile(labels.reshape(-1,1), (1,outputs.shape[1]))
    maxima = measurements.maximum_position(outputs, mask, np.arange(1, np.amax(mask)+1))
    if pos: return maxima
    return [c for (r,c) in maxima]

class Network:
    def predict(self,xs):
        """Prediction is the same as forward propagation."""
        return self.forward(xs)

class Softmax(Network):
    """A logistic regression network."""
    def __init__(self,Nh,No,initial_range=initial_range,rand=np.random.rand):
        self.Nh = Nh
        self.No = No
        self.W2 = randu(No,Nh+1)*initial_range
        self.DW2 = np.zeros((No,Nh+1))
    def ninputs(self):
        return self.Nh
    def noutputs(self):
        return self.No
    def forward(self,ys):
        n = len(ys)
        inputs,zs = [None]*n,[None]*n
        for i in range(n):
            inputs[i] = np.concatenate([np.ones(1),ys[i]])
            temp = np.dot(self.W2,inputs[i])<commit_msg>Add new back translation routine<commit_after>from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals
from builtins import zip
from builtins import range
from builtins import object

import unicodedata
import numpy as np

from scipy.ndimage import measurements
from scipy.special import expit

initial_range = 0.1


class Codec(object):
    """Translate between integer codes and characters."""
    def init(self, charset):
        charset = sorted(list(set(charset)))
        self.code2char = {}
        self.char2code = {}
        for code,char in enumerate(charset):
            self.code2char[code] = char
            self.char2code[char] = code
        return self
    def size(self):
        """The total number of codes (use this for the number of output
        classes when training a classifier."""
        return len(list(self.code2char.keys()))
    def encode(self, s):
        "Encode the string `s` into a code sequence."
        tab = self.char2code
        dflt = self.char2code["~"]
        return [self.char2code.get(c,dflt) for c in s]
    def decode(self, l):
        "Decode a code sequence into a string."
        s = [self.code2char.get(c,"~") for c in l]
        return s


def normalize_nfkc(s):
    return unicodedata.normalize('NFKC',s)


def prepare_line(line, pad=16):
    """Prepare a line for recognition; this inverts it, transposes
    it, and pads it."""
    line = line * 1.0/np.amax(line)
    line = np.amax(line)-line
    line = line.T
    if pad>0:
        w = line.shape[1]
        line = np.vstack([np.zeros((pad,w)),line,np.zeros((pad,w))])
    return line


def randu(*shape):
    # ATTENTION: whether you use randu or randn can make a difference.
    """Generate uniformly random values in the range (-1,1).
    This can usually be used as a drop-in replacement for `randn`
    resulting in a different distribution."""
    return 2*np.random.rand(*shape)-1


def sigmoid(x):
    """
    Compute the sigmoid function. We don't bother with clipping the input
    value because IEEE floating point behaves reasonably with this function
    even for infinities. 
    
    Further we use scipy's expit function which is ~50% faster for decently
    sized arrays.
    """
    return expit(x)

# These are the nonlinearities used by the LSTM network.
# We don't bother parameterizing them here

def ffunc(x):
    "Nonlinearity used for gates."
    return 1.0/(1.0+np.exp(-x))
def gfunc(x):
    "Nonlinearity used for input to state."
    return np.tanh(x)
# ATTENTION: try linear for hfunc
def hfunc(x):
    "Nonlinearity used for output."
    return np.tanh(x)

################################################################
# LSTM classification with forward/backward alignment ("CTC")
################################################################

def translate_back(outputs, threshold=0.5, pos=0):
    """Translate back. Thresholds on class 0, then assigns
    the maximum class to each region."""
    labels, n = measurements.label(outputs[:,0] < threshold)
    mask = np.tile(labels.reshape(-1,1), (1,outputs.shape[1]))
    maxima = measurements.maximum_position(outputs, mask, np.arange(1, np.amax(mask)+1))
    if pos: return maxima
    return [c for (r,c) in maxima if c != 0]

def translate_back_locations(outputs, threshold=0.5):
    """
    Translates back the network output to a class sequence.

    Thresholds on class 0, then assigns the maximum (non-zero) class to each
    region. Difference to translate_back is the output region not just the
    maximum's position is returned.

    Args:

    Returns:
        A list with tuples (class, start, end, max). max is the maximum value
        of the softmax layer in the region.
    """
    labels, n = measurements.label(outputs[:,0] < threshold)
    mask = np.tile(labels.reshape(-1,1), (1,outputs.shape[1]))
    maxima = measurements.maximum_position(outputs, mask, np.arange(1, np.amax(mask)+1))
    p = 0
    start = None
    x = []
    for idx, val in enumerate(labels):
        if val != 0 and start is None:
            start = idx
            p += 1
        if val == 0 and start:
            if maxima[p-1][1] == 0:
                start = None
            else:
                x.append((maxima[p-1][1], start, idx, maxima[p-1][0]))
                start = None
    return x


class Network:
    def predict(self,xs):
        """Prediction is the same as forward propagation."""
        return self.forward(xs)

class Softmax(Network):
    """A logistic regression network."""
    def __init__(self,Nh,No,initial_range=initial_range,rand=np.random.rand):
        self.Nh = Nh
        self.No = No
        self.W2 = randu(No,Nh+1)*initial_range
        self.DW2 = np.zeros((No,Nh+1))
    def ninputs(self):
        return self.Nh
    def noutputs(self):
        return self.No
    def forward(self,ys):
        n = len(ys)
        inputs,zs = [None]*n,[None]*n
        for i in range(n):
            inputs[i] = np.concatenate([np.ones(1),ys[i]])
            temp = np.dot(self.W2,inputs[i])
{'fraction_non_alphanumeric': 0.13835920177383593, 'fraction_numerical': 0.012195121951219513, 'mean_word_length': 2.906886097877869, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4848635', 'n_tokens_mistral': 2846, 'n_tokens_neox': 2738, 'n_words': 883}