<commit_before>import numpy as np

from vsm.corpus import Corpus
from vsm.corpus.util import *
from vsm.extensions.corpuscleanup import apply_stoplist_len
import os
import re


class CorpusSent(Corpus):
    """
    A subclass of Corpus whose purpose is to store original
    sentence information in the Corpus
    
    :See Also: :class: Corpus
    """
    def __init__(self, corpus, sentences, context_types=[], context_data=[], 
		remove_empty=False):
       
       super(CorpusSent, self).__init__(corpus, context_types=context_types,
		 context_data=context_data, remove_empty=remove_empty)
       
       sentences = [re.sub('\n', ' ', s) for s in sentences]
       self.sentences = np.array(sentences)


    def __set_words_int(self):
        """
        Mapping of words to their integer representations.
        """
        self.words_int = dict((t,i) for i,t in enumerate(self.words))


    def apply_stoplist(self, stoplist=[], freq=0):
        """ 
        Takes a Corpus object and returns a copy of it with words in the
        stoplist removed and with words of frequency <= `freq` removed.
        
	    :param stoplist: The list of words to be removed.
        :type stoplist: list
        
        :type freq: integer, optional
	    :param freq: A threshold where words of frequency <= 'freq' are 
            removed. Default is 0.
            
        :returns: Copy of corpus with words in the stoplist and words of
            frequnecy <= 'freq' removed.

        :See Also: :class:`Corpus`
        """
        if freq:
            #TODO: Use the TF model instead

            print 'Computing collection frequencies'
            cfs = np.zeros_like(self.words, dtype=self.corpus.dtype)
    
            for word in self.corpus:
                cfs[word] += 1

            print 'Selecting words of frequency <=', freq
            freq_stop = np.arange(cfs.size)[(cfs <= freq)]
            stop = set(freq_stop)
        else:
            stop = set()

        for t in stoplist:
            if t in self.words:
                stop.add(self.words_int[t])

        if not stop:
            print 'Stop list is empty.'
            return self
    
        print 'Removing stop words'
        f = np.vectorize(lambda x: x not in stop)
        corpus = self.corpus[f(self.corpus)]

        print 'Rebuilding corpus'
        corpus = [self.words[i] for i in corpus]
        context_data = []
        for i in xrange(len(self.context_data)):
            print 'Recomputing token breaks:', self.context_types[i]
            tokens = self.view_contexts(self.context_types[i])
            spans = [t[f(t)].size for t in tokens]
            tok = self.context_data[i].copy()
            tok['idx'] = np.cumsum(spans)
            context_data.append(tok)

        return CorpusSent(corpus, self.sentences, context_data=context_data,
                            context_types=self.context_types)


    @staticmethod
    def load(file):
        """
        Loads data into a Corpus object that has been stored using
        `save`.
        
        :param file: Designates the file to read. If `file` is a string ending
            in `.gz`, the file is first gunzipped. See `numpy.load`
            for further details.
        :type file: string-like or file-like object

        :returns: c : A Corpus object storing the data found in `file`.

        :See Also: :class: Corpus, :meth: Corpus.load, :meth: numpy.load
        """
        print 'Loading corpus from', file
        arrays_in = np.load(file)

        c = CorpusSent([], [])
        c.corpus = arrays_in['corpus']
        c.words = arrays_in['words']
        c.sentences = arrays_in['sentences']
        c.context_types = arrays_in['context_types'].tolist()

        c.context_data = list()
        for n in c.context_types:
            t = arrays_in['context_data_' + n]
            c.context_data.append(t)

        c.__set_words_int()

        return c

    def save(self, file):
        """
        Saves data from a CorpusSent object as an `npz` file.
        
        :param file: Designates the file to which to save data. See
            `numpy.savez` for further details.
        :type file: str-like or file-like object
            
        :returns: None

        :See Also: :class: Corpus, :meth: Corpus.save, :meth: numpy.savez
        """
	
	print 'Saving corpus as', file
        arrays_out = dict()
        arrays_out['corpus'] = self.corpus
        arrays_out['words'] = self.words
        arrays_out['sentences'] = self.sentences
        arrays_out['context_types'] = np.asarray(self.context_types)

        for i,t in enumerate(self.context_data):
            key = 'context_data_' + self.context_types[i]
            arrays_out[key] = t

        np.savez(file, **arrays_out)
        
    
    def sent_int(self, sent):
        """
        sent : list of strings
            List of sentence tokenization.
            The list could be a subset of existing sentence tokenization.
        """
        tok = self.view_contexts('sentence', as_strings=True)
        sent_li = []
        for i in xrange(len(tok)):
            sent_li.append(sent)
        keys = [i for i in xrange(len(tok)) 
                if set(sent_li[i]).issubset(tok[i].tolist())]
        
        n = len(keys)
        if n == 0:
            raise Exception('No token fits {0}.'.format(sent))
        elif n > 1:
            return keys
        return keys[0] 


def sim_sent_sent(ldaviewer, sent, print_len=10):
    """
    ldaviewer : ldaviewer object
    sent : sentence index or sentence as a list of words

    Returns
    -------
    sim_sents : numpy array
        (sentence index, probability) as (i, value) pair.
    tokenized_sents : list of arrays
        List containing tokenized sentences as arrays.
    orig_sents : list of strings
        List containing original sentences as strings.
    """
    from vsm.viewer.ldagibbsviewer import LDAGibbsViewer

    corp = ldaviewer.corpus
    ind = sent
    if isinstance(sent, list) and isinstance(sent[0], str):
        ind = corp.sent_int(sent)
    sim_sents = ldaviewer.sim_doc_doc(ind, print_len=print_len, as_strings=False)
    lc = sim_sents['i'][:print_len]
    
    # only returns print_len length<commit_msg>sim_sent_sent with as_strings=True as default.<commit_after>import numpy as np

from vsm.corpus import Corpus
from vsm.corpus.util import *
from vsm.extensions.corpuscleanup import apply_stoplist_len
import os
import re


class CorpusSent(Corpus):
    """
    A subclass of Corpus whose purpose is to store original
    sentence information in the Corpus
    
    :See Also: :class: Corpus
    """
    def __init__(self, corpus, sentences, context_types=[], context_data=[], 
		remove_empty=False):
       
       super(CorpusSent, self).__init__(corpus, context_types=context_types,
		 context_data=context_data, remove_empty=remove_empty)
       
       sentences = [re.sub('\n', ' ', s) for s in sentences]
       self.sentences = np.array(sentences)


    def __set_words_int(self):
        """
        Mapping of words to their integer representations.
        """
        self.words_int = dict((t,i) for i,t in enumerate(self.words))


    def apply_stoplist(self, stoplist=[], freq=0):
        """ 
        Takes a Corpus object and returns a copy of it with words in the
        stoplist removed and with words of frequency <= `freq` removed.
        
	    :param stoplist: The list of words to be removed.
        :type stoplist: list
        
        :type freq: integer, optional
	    :param freq: A threshold where words of frequency <= 'freq' are 
            removed. Default is 0.
            
        :returns: Copy of corpus with words in the stoplist and words of
            frequnecy <= 'freq' removed.

        :See Also: :class:`Corpus`
        """
        if freq:
            #TODO: Use the TF model instead

            print 'Computing collection frequencies'
            cfs = np.zeros_like(self.words, dtype=self.corpus.dtype)
    
            for word in self.corpus:
                cfs[word] += 1

            print 'Selecting words of frequency <=', freq
            freq_stop = np.arange(cfs.size)[(cfs <= freq)]
            stop = set(freq_stop)
        else:
            stop = set()

        for t in stoplist:
            if t in self.words:
                stop.add(self.words_int[t])

        if not stop:
            print 'Stop list is empty.'
            return self
    
        print 'Removing stop words'
        f = np.vectorize(lambda x: x not in stop)
        corpus = self.corpus[f(self.corpus)]

        print 'Rebuilding corpus'
        corpus = [self.words[i] for i in corpus]
        context_data = []
        for i in xrange(len(self.context_data)):
            print 'Recomputing token breaks:', self.context_types[i]
            tokens = self.view_contexts(self.context_types[i])
            spans = [t[f(t)].size for t in tokens]
            tok = self.context_data[i].copy()
            tok['idx'] = np.cumsum(spans)
            context_data.append(tok)

        return CorpusSent(corpus, self.sentences, context_data=context_data,
                            context_types=self.context_types)


    @staticmethod
    def load(file):
        """
        Loads data into a Corpus object that has been stored using
        `save`.
        
        :param file: Designates the file to read. If `file` is a string ending
            in `.gz`, the file is first gunzipped. See `numpy.load`
            for further details.
        :type file: string-like or file-like object

        :returns: c : A Corpus object storing the data found in `file`.

        :See Also: :class: Corpus, :meth: Corpus.load, :meth: numpy.load
        """
        print 'Loading corpus from', file
        arrays_in = np.load(file)

        c = CorpusSent([], [])
        c.corpus = arrays_in['corpus']
        c.words = arrays_in['words']
        c.sentences = arrays_in['sentences']
        c.context_types = arrays_in['context_types'].tolist()

        c.context_data = list()
        for n in c.context_types:
            t = arrays_in['context_data_' + n]
            c.context_data.append(t)

        c.__set_words_int()

        return c

    def save(self, file):
        """
        Saves data from a CorpusSent object as an `npz` file.
        
        :param file: Designates the file to which to save data. See
            `numpy.savez` for further details.
        :type file: str-like or file-like object
            
        :returns: None

        :See Also: :class: Corpus, :meth: Corpus.save, :meth: numpy.savez
        """
	
	print 'Saving corpus as', file
        arrays_out = dict()
        arrays_out['corpus'] = self.corpus
        arrays_out['words'] = self.words
        arrays_out['sentences'] = self.sentences
        arrays_out['context_types'] = np.asarray(self.context_types)

        for i,t in enumerate(self.context_data):
            key = 'context_data_' + self.context_types[i]
            arrays_out[key] = t

        np.savez(file, **arrays_out)
        
    
    def sent_int(self, sent):
        """
        sent : list of strings
            List of sentence tokenization.
            The list could be a subset of existing sentence tokenization.
        """
        tok = self.view_contexts('sentence', as_strings=True)
        sent_li = []
        for i in xrange(len(tok)):
            sent_li.append(sent)
        keys = [i for i in xrange(len(tok)) 
                if set(sent_li[i]).issubset(tok[i].tolist())]
        
        n = len(keys)
        if n == 0:
            raise Exception('No token fits {0}.'.format(sent))
        elif n > 1:
            return keys
        return keys[0] 


def sim_sent_sent(ldaviewer, sent, print_len=10):
    """
    ldaviewer : ldaviewer object
    sent : sentence index or sentence as a list of words

    Returns
    -------
    sim_sents : numpy array
        (sentence index, probability) as (i, value) pair.
    tokenized_sents : list of arrays
        List containing tokenized sentences as arrays.
    orig_sents : list of strings
        List containing original sentences as strings.
    """
    from vsm.viewer.ldagibbsviewer import LDAGibbsViewer

    corp = ldaviewer.corpus
    ind = sent
    if isinstance(sent, list) and isinstance(sent[0], str):
        ind = corp.sent_int(sent)
    sim_sents = ldaviewer.sim_doc_doc(ind, print_len=print_len)
    lc = sim_sents['doc'][:print_len]
    lc = [s.split(', ') for s in lc]
    lc = [int(s[-1]) for s in lc]
    
    # only returns print_len length
{'fraction_non_alphanumeric': 0.09907687410472704, 'fraction_numerical': 0.0016711761897182874, 'mean_word_length': 2.025276841598459, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3493447', 'n_tokens_mistral': 3861, 'n_tokens_neox': 3712, 'n_words': 1112}