<commit_before>import sys; import os
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('.'))

from numpy import zeros
from Utils.DataSource import *
import nltk.corpus
from Keywords_Wordnet.KeywordExtractor import *
import re
from collections import Counter
from sklearn.feature_extraction.text import TfidfTransformer
import io
from Utils.Globals import *


class MatrixCreator:

    def __init__(self):
        """
        Initialize class variables
        :return: None
        """
        self.ds = DataSource()
        self.ids = []
        self.num_entries = 0
        self.num_articles = 0
        self.num_article_words = 0
        self.article_titles = []
        self.stopwords = set(nltk.corpus.stopwords.words('english'))
        self.lemmatizer = WordNetLemmatizer()


    def get_num_entries(self):
        """
        Gets the number of non-zero entries in the matrix.
        :return: number of non-zero entries in the matrix
        """
        return self.num_entries

    def get_num_articles(self):
        """
        Gets the number of articles, which is the number of rows in the matrix
        :return: number of articles
        """
        return self.num_articles


    def get_num_article_words(self):
        """
        Gets the number of unique words across all documents
        :return: number of unique words across all documents
        """
        return self.num_article_words

    def get_article_titles(self):
        """
        Gets ordered list of article titles corresponding to
        article ids in self.ids
        :return: list of article titles
        """
        return self.article_titles

    def get_article_ids(self):
        """
        Gets article ids
        :return: list of article ids
        """
        return self.ids

    def retrieve_article_ids_titles_filenames(self):
        """
        Retrieves article ids, titles, and filenames from database
        and sets class variables
        :return: None
        """
        articles = self.ds.get_article_ids_titles_filenames()
        self.ids = []
        self.article_titles = []
        self.filenames = []
        for article in articles:
            if os.path.isfile("articles/{}".format(article[2])):
                self.ids.append(article[0])
                self.article_titles.append(article[1])
                self.filenames.append(article[2])
        self.num_articles = len(self.article_titles)

    def get_article_text_by_article(self):
        """
        Gets list of sets of words by article, along with set of keywords
        across all articles
        :return: Set of words used in all articles
        """
        pattern = re.compile(r'TITLE:(.*)TEXT:(.*)', re.DOTALL)

        self.article_words_by_article = []<commit_msg>Update MatrixCreator.py<commit_after>import sys; import os
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('.'))

from numpy import zeros
from Utils.DataSource import *
import nltk.corpus
from Keywords_Wordnet.KeywordExtractor import *
import re
from collections import Counter
from sklearn.feature_extraction.text import TfidfTransformer
import io
from Utils.Globals import *


class MatrixCreator:

    def __init__(self):
        """
        Initialize class variables
        :return: None
        """
        self.ds = DataSource()
        self.ids = []
        self.num_entries = 0
        self.num_articles = 0
        self.num_article_words = 0
        self.article_titles = []
        self.stopwords = set(nltk.corpus.stopwords.words('english'))
        self.lemmatizer = WordNetLemmatizer()


    def get_num_entries(self):
        """
        Gets the number of non-zero entries in the matrix.
        :return: number of non-zero entries in the matrix
        """
        return self.num_entries

    def get_num_articles(self):
        """
        Gets the number of articles, which is the number of rows in the matrix
        :return: number of articles
        """
        return self.num_articles


    def get_num_article_words(self):
        """
        Gets the number of unique words across all documents
        :return: number of unique words across all documents
        """
        return self.num_article_words

    def get_article_titles(self):
        """
        Gets ordered list of article titles corresponding to
        article ids in self.ids
        :return: list of article titles
        """
        return self.article_titles

    def get_article_ids(self):
        """
        Gets article ids
        :return: list of article ids
        """
        return self.ids

    def retrieve_article_ids_titles_filenames(self):
        """
        Retrieves article ids, titles, and filenames from database
        and sets class variables
        :return: None
        """
        articles = self.ds.get_article_ids_titles_filenames()
        self.ids = []
        self.article_titles = []
        self.filenames = []
        for article in articles:
            if os.path.isfile(articles_path + article[2]):
                self.ids.append(article[0])
                self.article_titles.append(article[1])
                self.filenames.append(article[2])
        self.num_articles = len(self.article_titles)

    def get_article_text_by_article(self):
        """
        Gets list of sets of words by article, along with set of keywords
        across all articles
        :return: Set of words used in all articles
        """
        pattern = re.compile(r'TITLE:(.*)TEXT:(.*)', re.DOTALL)

        self.article_words_by_article = []
{'fraction_non_alphanumeric': 0.08863473909935668, 'fraction_numerical': 0.0032165832737669764, 'mean_word_length': 2.214819069500287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4862701', 'n_tokens_mistral': 1608, 'n_tokens_neox': 1568, 'n_words': 436}