<commit_before># coding: utf-8
from gensim import matutils
from nltk.corpus import stopwords
from nltk import wordpunct_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import networkx as nx
import itertools
import pandas
from networkx.readwrite import json_graph
from scipy import spatial
import random

__author__ = "<NAME>, <NAME>"
__email__ = "<EMAIL>"


class Corpus:

    def __init__(self,
                 source_file_path,
                 language=None,
                 vectorization='tfidf',
                 max_relative_frequency=0.95,
                 min_absolute_frequency=2,
                 preprocessor=None,
                 sample=None):

        self._source_file_path = source_file_path
        self._language = language
        self._vectorization = vectorization
        self._max_relative_frequency = max_relative_frequency
        self._min_absolute_frequency = min_absolute_frequency
        self._preprocessor = preprocessor

        self.data_frame = pandas.read_csv(source_file_path, sep='\t', encoding='utf-8')
        if sample:
            self.data_frame = self.data_frame.sample(frac=0.8)
        self.data_frame.fillna(' ')
        self.size = self.data_frame.count(0)[0]
        if preprocessor is not None:
            for i in self.data_frame.index.tolist():
                full_content = self.data_frame.iloc[i]['full_content']
                self.data_frame.loc[i, 'full_content'] = preprocessor.process_sentence(full_content)
        stop_words = []
        if language is not None:
            stop_words = stopwords.words(language)
        if vectorization == 'tfidf':
            self.vectorizer = TfidfVectorizer(max_df=max_relative_frequency,
                                              min_df=min_absolute_frequency,
                                              max_features=2000,
                                              stop_words=stop_words)
        elif vectorization == 'tf':
            self.vectorizer = CountVectorizer(max_df=max_relative_frequency,
                                              min_df=min_absolute_frequency,
                                              max_features=2000,
                                              stop_words=stop_words)
        else:
            raise ValueError('Unknown vectorization type: %s' % vectorization)
        self.sklearn_vector_space = self.vectorizer.fit_transform(self.data_frame['full_content'].tolist())
        self.gensim_vector_space = matutils.Sparse2Corpus(self.sklearn_vector_space, documents_columns=False)
        vocab = self.vectorizer.get_feature_names()
        self.vocabulary = dict([(i, s) for i, s in enumerate(vocab)])

    def export(self, file_path):
        self.data_frame.to_csv(path_or_buf=file_path, sep='\t', encoding='utf-8')

    def full_content(self, doc_id):
        return self.data_frame.iloc[doc_id]['full_content']

    def short_content(self, doc_id):
        return self.data_frame.iloc[doc_id]['short_content']

    def date(self, doc_id):
        return self.data_frame.iloc[doc_id]['date']

    def authors(self, doc_id):
        return unicode(self.data_frame.iloc[doc_id]['authors']).split(', ')

    def affiliations(self, doc_id):
        return unicode(self.data_frame.iloc[doc_id]['affiliations']).split(', ')

    def documents_by_author(self, author, date=None):
        ids = []
        potential_ids = range(self.size)
        if date:
            potential_ids = self.doc_ids(date)
        for i in potential_ids:
            if self.is_author(author, i):
                ids.append(i)
        return ids

    def is_author(self, author, doc_id):
        return author in self.authors(doc_id)

    def docs_for_word(self, word_id):
        ids = []
        for i in range(self.size):
            vector = self.vector_for_document(i)
            if vector[word_id] > 0:
                ids.append(i)
        return ids

    def doc_ids(self, date):
        return self.data_frame[self.data_frame['date'] == date].index.tolist()

    def vector_for_document(self, doc_id):
        vector = self.sklearn_vector_space[doc_id]
        cx = vector.tocoo()
        weights = [0.0] * len(self.vocabulary)
        for row, word_id, weight in itertools.izip(cx.row, cx.col, cx.data):
            weights[word_id] = weight
        return weights

    def word_for_id(self, word_id):
        return self.vocabulary.get(word_id)

    def similar_documents(self, doc_id, num_docs):
        doc_weights = self.vector_for_document(doc_id)
        similarities = []
        for a_doc_id in range(self.size):
            if a_doc_id != doc_id:
                similarity = 1.0 - spatial.distance.cosine(doc_weights, self.vector_for_document(a_doc_id))
                similarities.append((a_doc_id, similarity))
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:num_docs]

    def collaboration_network(self, doc_ids=None):
        nx_graph = nx.Graph(name='')
        for doc_id in doc_ids:
            authors = self.authors(doc_id)
            for author in authors:
                nx_graph.add_node(author)
            for i in range(0, len(authors)):
                for j in range(i+1, len(authors)):
                    nx_graph.add_edge(authors[i], authors[j])
        bb = nx.betweenness_centrality(nx_graph)
        nx.set_node_attributes(nx_graph, 'betweenness', bb)
        return json_graph.node_link_data(nx_graph)<commit_msg>Add option to export the collaboration network in NetworkX format<commit_after># coding: utf-8
from gensim import matutils
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import networkx as nx
import itertools
import pandas
from networkx.readwrite import json_graph
from scipy import spatial

__author__ = "<NAME>, <NAME>"
__email__ = "<EMAIL>"


class Corpus:

    def __init__(self,
                 source_file_path,
                 language=None,
                 vectorization='tfidf',
                 max_relative_frequency=0.95,
                 min_absolute_frequency=2,
                 preprocessor=None,
                 sample=None):

        self._source_file_path = source_file_path
        self._language = language
        self._vectorization = vectorization
        self._max_relative_frequency = max_relative_frequency
        self._min_absolute_frequency = min_absolute_frequency
        self._preprocessor = preprocessor

        self.data_frame = pandas.read_csv(source_file_path, sep='\t', encoding='utf-8')
        if sample:
            self.data_frame = self.data_frame.sample(frac=0.8)
        self.data_frame.fillna(' ')
        self.size = self.data_frame.count(0)[0]
        if preprocessor is not None:
            for i in self.data_frame.index.tolist():
                full_content = self.data_frame.iloc[i]['full_content']
                self.data_frame.loc[i, 'full_content'] = preprocessor.process_sentence(full_content)
        stop_words = []
        if language is not None:
            stop_words = stopwords.words(language)
        if vectorization == 'tfidf':
            self.vectorizer = TfidfVectorizer(max_df=max_relative_frequency,
                                              min_df=min_absolute_frequency,
                                              max_features=2000,
                                              stop_words=stop_words)
        elif vectorization == 'tf':
            self.vectorizer = CountVectorizer(max_df=max_relative_frequency,
                                              min_df=min_absolute_frequency,
                                              max_features=2000,
                                              stop_words=stop_words)
        else:
            raise ValueError('Unknown vectorization type: %s' % vectorization)
        self.sklearn_vector_space = self.vectorizer.fit_transform(self.data_frame['full_content'].tolist())
        self.gensim_vector_space = matutils.Sparse2Corpus(self.sklearn_vector_space, documents_columns=False)
        vocab = self.vectorizer.get_feature_names()
        self.vocabulary = dict([(i, s) for i, s in enumerate(vocab)])

    def export(self, file_path):
        self.data_frame.to_csv(path_or_buf=file_path, sep='\t', encoding='utf-8')

    def full_content(self, doc_id):
        return self.data_frame.iloc[doc_id]['full_content']

    def short_content(self, doc_id):
        return self.data_frame.iloc[doc_id]['short_content']

    def date(self, doc_id):
        return self.data_frame.iloc[doc_id]['date']

    def authors(self, doc_id):
        return unicode(self.data_frame.iloc[doc_id]['authors']).split(', ')

    def affiliations(self, doc_id):
        return unicode(self.data_frame.iloc[doc_id]['affiliations']).split(', ')

    def documents_by_author(self, author, date=None):
        ids = []
        potential_ids = range(self.size)
        if date:
            potential_ids = self.doc_ids(date)
        for i in potential_ids:
            if self.is_author(author, i):
                ids.append(i)
        return ids

    def is_author(self, author, doc_id):
        return author in self.authors(doc_id)

    def docs_for_word(self, word_id):
        ids = []
        for i in range(self.size):
            vector = self.vector_for_document(i)
            if vector[word_id] > 0:
                ids.append(i)
        return ids

    def doc_ids(self, date):
        return self.data_frame[self.data_frame['date'] == date].index.tolist()

    def vector_for_document(self, doc_id):
        vector = self.sklearn_vector_space[doc_id]
        cx = vector.tocoo()
        weights = [0.0] * len(self.vocabulary)
        for row, word_id, weight in itertools.izip(cx.row, cx.col, cx.data):
            weights[word_id] = weight
        return weights

    def word_for_id(self, word_id):
        return self.vocabulary.get(word_id)

    def similar_documents(self, doc_id, num_docs):
        doc_weights = self.vector_for_document(doc_id)
        similarities = []
        for a_doc_id in range(self.size):
            if a_doc_id != doc_id:
                similarity = 1.0 - spatial.distance.cosine(doc_weights, self.vector_for_document(a_doc_id))
                similarities.append((a_doc_id, similarity))
        similarities.sort(key=lambda x: x[1], reverse=True)
        return similarities[:num_docs]

    def collaboration_network(self, doc_ids=None, nx_format=False):
        nx_graph = nx.Graph(name='')
        for doc_id in doc_ids:
            authors = self.authors(doc_id)
            for author in authors:
                nx_graph.add_node(author)
            for i in range(0, len(authors)):
                for j in range(i+1, len(authors)):
                    nx_graph.add_edge(authors[i], authors[j])
        bb = nx.betweenness_centrality(nx_graph)
        nx.set_node_attributes(nx_graph, 'betweenness', bb)
        if nx_format:
            return nx_graph
        else:
            return json_graph.node_link_data(nx_graph)
{'fraction_non_alphanumeric': 0.09531575609314125, 'fraction_numerical': 0.005073842529672919, 'mean_word_length': 2.250294464075383, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515882', 'n_tokens_mistral': 3324, 'n_tokens_neox': 3210, 'n_words': 622}