<commit_before># Cooccurrence matrix construction tools
# for fitting the GloVe model.

import cPickle as pickle

from corpus_cython import construct_cooccurrence_matrix


class Corpus(object):
    """
    Class for constructing a cooccurrence matrix
    from a corpus.
    """
    
    def __init__(self):

        self.dictionary = None
        self.matrix = None

    def fit_dictionary(self, corpus):
        """
        Perform a pass through the corpus to
        construct a word-id mapping.
        
        Parameters:
        - iterable of lists of strings corpus
        """

        dictionary = {}

        # First pass to construct the dictionary
        for context in corpus:
            for word in context:
                if word not in dictionary:
                    dictionary[word] = len(dictionary)

        self.dictionary = dictionary

    def fit_matrix(self, corpus, window=10):
        """
        Perform a pass through the corpus to construct
        the cooccurrence matrix. 

        You must call fit_dictionary first.

        Parameters:
        - iterable of lists of strings corpus
        - int window: the length of the (symmetric)
          context window used for cooccurrence.
        """
        if self.dictionary is None:
            raise Exception('You must fit the dictionary before transforming the corpus')

        self.matrix = construct_coocurrence_matrix(corpus, 
                                                   self.dictionary, int(window))

    def save(self, filename):
        
        with open(filename, 'wb') as savefile:
            pickle.dump((self.dictionary, self.matrix),
                        savefile,
                        protocol=pickle.HIGHEST_PROTOCOL)

    @classmethod
    def load(cls, filename):

        instance = cls()

        with open(filename, 'rb') as savefile:
            instance.dictionary, instance.matrix = pickle.load(savefile)

        return instance<commit_msg>Typo fix.<commit_after># Cooccurrence matrix construction tools
# for fitting the GloVe model.

import cPickle as pickle

from corpus_cython import construct_cooccurrence_matrix


class Corpus(object):
    """
    Class for constructing a cooccurrence matrix
    from a corpus.
    """
    
    def __init__(self):

        self.dictionary = None
        self.matrix = None

    def fit_dictionary(self, corpus):
        """
        Perform a pass through the corpus to
        construct a word-id mapping.
        
        Parameters:
        - iterable of lists of strings corpus
        """

        dictionary = {}

        # First pass to construct the dictionary
        for context in corpus:
            for word in context:
                if word not in dictionary:
                    dictionary[word] = len(dictionary)

        self.dictionary = dictionary

    def fit_matrix(self, corpus, window=10):
        """
        Perform a pass through the corpus to construct
        the cooccurrence matrix. 

        You must call fit_dictionary first.

        Parameters:
        - iterable of lists of strings corpus
        - int window: the length of the (symmetric)
          context window used for cooccurrence.
        """
        if self.dictionary is None:
            raise Exception('You must fit the dictionary before transforming the corpus')

        self.matrix = construct_cooccurrence_matrix(corpus, 
                                                   self.dictionary, int(window))

    def save(self, filename):
        
        with open(filename, 'wb') as savefile:
            pickle.dump((self.dictionary, self.matrix),
                        savefile,
                        protocol=pickle.HIGHEST_PROTOCOL)

    @classmethod
    def load(cls, filename):

        instance = cls()

        with open(filename, 'rb') as savefile:
            instance.dictionary, instance.matrix = pickle.load(savefile)

        return instance
{'fraction_non_alphanumeric': 0.06646294881588999, 'fraction_numerical': 0.0010185892538833714, 'mean_word_length': 1.8361010830324909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '586950', 'n_tokens_mistral': 1030, 'n_tokens_neox': 930, 'n_words': 330}