def collate_tokens(values, pad_idx, eos_idx=None, left_pad=False, move_eos_to_beginning=False, pad_to_length=None, pad_to_multiple=1, pad_to_bsz=None):
    'Convert a list of 1d tensors into a padded 2d tensor.'
    size = max((v.size(0) for v in values))
    size = (size if (pad_to_length is None) else max(size, pad_to_length))
    if ((pad_to_multiple != 1) and ((size % pad_to_multiple) != 0)):
        size = int(((((size - 0.1) // pad_to_multiple) + 1) * pad_to_multiple))
    batch_size = (len(values) if (pad_to_bsz is None) else max(len(values), pad_to_bsz))
    res = values[0].new(batch_size, size).fill_(pad_idx)

    def copy_tensor(src, dst):
        assert (dst.numel() == src.numel())
        if move_eos_to_beginning:
            if (eos_idx is None):
                dst[0] = src[(- 1)]
            else:
                dst[0] = eos_idx
            dst[1:] = src[:(- 1)]
        else:
            dst.copy_(src)
    for (i, v) in enumerate(values):
        copy_tensor(v, (res[i][(size - len(v)):] if left_pad else res[i][:len(v)]))
    return res
Convert a list of 1d tensors into a padded 2d tensor.
def collate_tokens(values, pad_idx, eos_idx=None, left_pad=False, move_eos_to_beginning=False, pad_to_length=None, pad_to_multiple=1, pad_to_bsz=None):
    
    size = max((v.size(0) for v in values))
    size = (size if (pad_to_length is None) else max(size, pad_to_length))
    if ((pad_to_multiple != 1) and ((size % pad_to_multiple) != 0)):
        size = int(((((size - 0.1) // pad_to_multiple) + 1) * pad_to_multiple))
    batch_size = (len(values) if (pad_to_bsz is None) else max(len(values), pad_to_bsz))
    res = values[0].new(batch_size, size).fill_(pad_idx)

    def copy_tensor(src, dst):
        assert (dst.numel() == src.numel())
        if move_eos_to_beginning:
            if (eos_idx is None):
                dst[0] = src[(- 1)]
            else:
                dst[0] = eos_idx
            dst[1:] = src[:(- 1)]
        else:
            dst.copy_(src)
    for (i, v) in enumerate(values):
        copy_tensor(v, (res[i][(size - len(v)):] if left_pad else res[i][:len(v)]))
    return res