<commit_before>import csv
import os
import warnings

class CommentReader(object):

    __delimiter = ''
    __commentsFilepath = ""
    __commentsFile = None
    __commentReader = None
    
    __articlesFilepath = None
    __articlesFile = None
    __articlesReader = None
    __currentArticle = None

    __authorsFilepath = None
    __authorsFile = None
    __authorsReader = None
    __authors = None


    def __init__(self, commentsFilepath, arcticlesFilepath, authorsFilepath, delimiter=','):
        self.__delimiter = delimiter
        self.__commentsFilepath = commentsFilepath
        self.__articlesFilepath = arcticlesFilepath
        self.__authorsFilepath = authorsFilepath

    def open(self):
        if not os.path.exists(os.path.dirname(self.__commentsFilepath)):
            raise Exception("file not found!")

        self.__commentsFile = open(self.__commentsFilepath, 'r', newline='')
        self.__commentReader = csv.reader(self.__commentsFile, delimiter=self.__delimiter)

        self.__articlesFile = open(self.__articlesFilepath, 'r', newline='')
        self.__articlesReader = csv.reader(self.__articlesFile, delimiter=self.__delimiter)

        self.__authorsFile = open(self.__authorsFilepath, 'r', newline='')
        self.__authorsReader = csv.reader(self.__authorsFile, delimiter=self.__delimiter)

        return self


    def close(self):
        self.__commentsFile.close()
        self.__articlesFile.close()
        self.__authorsFile.close()


    def readAllData(self):
        raise DeprecationWarning("deprecated and not longer supported")

    
    def __loadAuthors(self):
        self.__authors = {}
        self.__authorsFile.seek(0)
        fileFormat = next(self.__authorsReader)

        for row in self.__authorsReader: # todo: don't load each author name. instead load seek offsets for each author id (less memory consumption). With delta encoding if necessary
            self.__authors[row[0]] = row[1]




    def __parseRow(self, row):
        raise DeprecationWarning("deprecated and not longer supported")

    def __parseIterRow(self, row):
        if not self.__authors:
            self.__loadAuthors()

        commentId = row[0]
        articleId = row[1]
        author = self.__authors[row[2]]
        text = row[3].replace("\\n", "\n")
        timestamp = row[4]
        parentId = row[5]
        upvotes = int(row[6])
        downvotes = int(row[7])

        if not self.__currentArticle:
            self.__articlesFile.seek(0)
            articlesFormat = next(self.__articlesReader)
            self.__currentArticle = next(self.__articlesReader)

        while self.__currentArticle[0] is not articleId:
            self.__currentArticle = next(self.__articlesReader)
        articleUrl = self.__currentArticle[1]

        return {
            "commentId": commentId,
            "article_url": articleUrl,
            "article_id": articleId,
            "comment_author": author,
            "comment_text" : text,
            "timestamp" : timestamp,
            "parent_comment_id" : parentId,
            "upvotes" : upvotes,
            "downvotes": downvotes
        }


    def __iter__(self):
        self.__commentsFile.seek(0)
        self.__commentReader.__iter__()
        # skip csv header in iteration mode:
        self.__commentReader.__next__()
        return self


    def __next__(self):
        return self.__parseIterRow(self.__commentReader.__next__())


    def __enter__(self):
        return self.open()


    def __exit__(self, type, value, traceback):<commit_msg>commentreader quickfix: endless loop, datatype issue<commit_after>import csv
import os
import warnings

class CommentReader(object):

    __delimiter = ''

    __commentsFilepath = ""
    __commentsFile = None
    __commentReader = None

    __articlesFilepath = None
    __articlesFile = None
    __articlesReader = None
    __currentArticle = None

    __authorsFilepath = None
    __authorsFile = None
    __authorsReader = None
    __authors = None


    def __init__(self, commentsFilepath, arcticlesFilepath, authorsFilepath, delimiter=','):
        self.__delimiter = delimiter

        self.__commentsFilepath = commentsFilepath
        self.__commentsFile = None
        self.__commentReader = None

        self.__articlesFilepath = arcticlesFilepath
        self.__articlesFile = None
        self.__articlesReader = None
        self.__currentArticle = None

        self.__authorsFilepath = authorsFilepath
        self.__authorsFile = None
        self.__authorsReader = None
        self.__authors = None


    def open(self):
        if (not os.path.exists(os.path.dirname(self.__commentsFilepath)) or
                not os.path.exists(os.path.dirname(self.__articlesFilepath)) or
                not os.path.exists(os.path.dirname(self.__authorsFilepath))):
            raise Exception("file not found!")

        self.__commentsFile = open(self.__commentsFilepath, 'r', newline='')
        self.__commentReader = csv.reader(self.__commentsFile, delimiter=self.__delimiter)

        self.__articlesFile = open(self.__articlesFilepath, 'r', newline='')
        self.__articlesReader = csv.reader(self.__articlesFile, delimiter=self.__delimiter)

        self.__authorsFile = open(self.__authorsFilepath, 'r', newline='')
        self.__authorsReader = csv.reader(self.__authorsFile, delimiter=self.__delimiter)

        return self


    def close(self):
        self.__commentsFile.close()
        self.__articlesFile.close()
        self.__authorsFile.close()


    def readAllData(self):
        raise DeprecationWarning("deprecated and not longer supported")

    
    def __loadAuthors(self):
        self.__authors = {}
        self.__authorsFile.seek(0)
        _ = next(self.__authorsReader)

        for row in self.__authorsReader: # todo: don't load each author name. instead load seek offsets for each author id (less memory consumption). With delta encoding if necessary
            self.__authors[row[0]] = row[1]


    def __parseRow(self, row):
        raise DeprecationWarning("deprecated and not longer supported")


    def __parseIterRow(self, row):
        if not self.__authors:
            self.__loadAuthors()

        commentId = row[0]
        articleId = row[1]
        author = self.__authors[row[2]]
        text = row[3].replace("\\n", "\n")
        timestamp = row[4]
        parentId = row[5]
        upvotes = int(row[6])
        downvotes = int(row[7])

        if not self.__currentArticle:
            self.__articlesFile.seek(0)
            _ = next(self.__articlesReader)
            self.__currentArticle = next(self.__articlesReader)

        while int(self.__currentArticle[0]) is not int(articleId):
            self.__currentArticle = next(self.__articlesReader)
        articleUrl = self.__currentArticle[1]

        return {
            "commentId": commentId,
            "article_url": articleUrl,
            "article_id": articleId,
            "comment_author": author,
            "comment_text" : text,
            "timestamp" : timestamp,
            "parent_comment_id" : parentId,
            "upvotes" : upvotes,
            "downvotes": downvotes
        }


    def __iter__(self):
        del self.__currentArticle
        self.__commentsFile.seek(0)
        self.__commentReader.__iter__()
        # skip csv header in iteration mode:
        self.__commentReader.__next__()
        return self


    def __next__(self):
        return self.__parseIterRow(self.__commentReader.__next__())


    def __enter__(self):
        return self.open()


    def __exit__(self, type, value, traceback):
{'fraction_non_alphanumeric': 0.09442116291251965, 'fraction_numerical': 0.00392875851231011, 'mean_word_length': 2.547143520668834, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '727004', 'n_tokens_mistral': 2191, 'n_tokens_neox': 2024, 'n_words': 462}