<commit_before>#
# Code for parsing talk forum data into something that can be processed further
# and/or used by a bot.
#
# 12/9/2014: <EMAIL>
#
import nltk
import csv
import sys

class TalkParser(object):
    def __init__(self):
        self.stopset = set(nltk.corpus.stopwords.words('english'))

    def parsecsv(self, fname, upperbound = None):
        """Parse CSV file containing talk data.  This should be replaceable
           with something that talks to the backing database containing the
           actual data.
        """
        csv.field_size_limit(sys.maxsize)

        with open(fname, 'r') as csvfile:
            talkreader = csv.reader(csvfile)

            rownum = 0
            header = []
            results = []
            
            for row in talkreader:
                result = {}
                if rownum == 0:
                    header = row
                elif rownum > upperbound:
                    break
                else:
                    for i,column in enumerate(header):
                        result[column] = row[i]
                        
                    result['text'] = self.parsebody(result['body'])

                    results.append(result)
                rownum += 1
            return results
        
    def parsebody(self, text):
        """Parse the body of a talk comment.  Hashtags are identified, and then
           the text is tokenized and stopwords are removed.
        """
        hashtags = []
        tokens_no_hashtags = []

        # tokenize the text, all lowercase
        tokens = nltk.WordPunctTokenizer().tokenize(text.lower())
<commit_msg>fix for bug when upperbound is None<commit_after>#
# Code for parsing talk forum data into something that can be processed further
# and/or used by a bot.
#
# 12/9/2014: <EMAIL>
#
import nltk
import csv
import sys

class TalkParser(object):
    def __init__(self):
        self.stopset = set(nltk.corpus.stopwords.words('english'))

    def parsecsv(self, fname, upperbound = None):
        """Parse CSV file containing talk data.  This should be replaceable
           with something that talks to the backing database containing the
           actual data.
        """
        csv.field_size_limit(sys.maxsize)

        with open(fname, 'r') as csvfile:
            talkreader = csv.reader(csvfile)

            rownum = 0
            header = []
            results = []
            
            for row in talkreader:
                result = {}
                if rownum == 0:
                    header = row
                elif upperbound is not None and rownum > upperbound:
                    break
                else:
                    for i,column in enumerate(header):
                        result[column] = row[i]
                        
                    result['text'] = self.parsebody(result['body'])

                    results.append(result)
                rownum += 1
            return results
        
    def parsebody(self, text):
        """Parse the body of a talk comment.  Hashtags are identified, and then
           the text is tokenized and stopwords are removed.
        """
        hashtags = []
        tokens_no_hashtags = []

        # tokenize the text, all lowercase
        tokens = nltk.WordPunctTokenizer().tokenize(text.lower())

{'fraction_non_alphanumeric': 0.08200180886343081, 'fraction_numerical': 0.006029544769369913, 'mean_word_length': 1.5483870967741935, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '573309', 'n_tokens_mistral': 902, 'n_tokens_neox': 843, 'n_words': 275}