<commit_before>import csv
import torch


def vectors_from_csv_file(file_like_object, unk_tensor=None):
    r"""Create a Vectors object from a csv file like object.

    Args:
        file_like_object (FileObject): a file like object to read data from.
        unk_tensor (int): a 1d tensors representing the vector associated with an unknown token
    Returns:
        Vectors: a Vectors object.
    """
    readCSV = csv.reader(file_like_object, delimiter=',')

    tokens = []
    vectors = []
    for row in readCSV:
        tokens.append(row[0])
        vectors.append(torch.tensor([float(c) for c in row[1].split()], dtype=torch.float))

    return Vectors(tokens, vectors, unk_tensor=unk_tensor)


class Vectors(object):
    r"""Creates a vectors object which maps tokens to vectors.

    Arguments:
        tokens (List[str]): a list of tokens.
        vectors (List[torch.Tensor]): a list of 1d tensors representing the vector associated with each token.
        unk_tensor (torch.Tensor): a 1d tensors representing the vector associated with an unknown token.

    """

    def __init__(self, tokens, vectors, unk_tensor=None):
        if unk_tensor is None and not vectors:
            raise ValueError("The vectors list is empty and a default unk_tensor wasn't provided.")

        unk_tensor = unk_tensor if unk_tensor is not None else torch.zeros(vectors[0].size(), dtype=torch.float)
        float_vectors = [vector.float() for vector in vectors]
        self.vectors = torch.classes.torchtext.Vectors(tokens, float_vectors, unk_tensor)<commit_msg>Resolving PR comments<commit_after>import csv
import torch


def vectors_from_csv_file(file_like_object, unk_tensor=None):
    r"""Create a Vectors object from a csv file like object.

    Args:
        file_like_object (FileObject): a file like object to read data from.
        unk_tensor (int): a 1d tensors representing the vector associated with an unknown token
    
    Returns:
        Vectors: a Vectors object.

    """
    readCSV = csv.reader(file_like_object, delimiter=',')

    tokens = []
    vectors = []
    for row in readCSV:
        tokens.append(row[0])
        vectors.append(torch.tensor([float(c) for c in row[1].split()], dtype=torch.float))

    return Vectors(tokens, vectors, unk_tensor=unk_tensor)


class Vectors(object):
    r"""Creates a vectors object which maps tokens to vectors.

    Arguments:
        tokens (List[str]): a list of tokens.
        vectors (List[torch.Tensor]): a list of 1d tensors representing the vector associated with each token.
        unk_tensor (torch.Tensor): a 1d tensors representing the vector associated with an unknown token.

    Raises:

        ValueError: if `vectors` is empty and a default `unk_tensor` isn't provided.
        RuntimeError: if `tokens` and `vectors` have different sizes or `tokens` has duplicates.

    """

    def __init__(self, tokens, vectors, unk_tensor=None):
        if unk_tensor is None and not vectors:
            raise ValueError("The vectors list is empty and a default unk_tensor wasn't provided.")

        unk_tensor = unk_tensor if unk_tensor is not None else torch.zeros(vectors[0].size(), dtype=torch.float)
        float_vectors = [vector.float() for vector in vectors]
        self.vectors = torch.classes.torchtext.Vectors(tokens, float_vectors, unk_tensor)
{'fraction_non_alphanumeric': 0.08878224355128975, 'fraction_numerical': 0.003599280143971206, 'mean_word_length': 3.303225806451613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3726169', 'n_tokens_mistral': 1004, 'n_tokens_neox': 951, 'n_words': 335}