<commit_before>from datasource import datasource
import csv
import os

class csvdatasource(datasource):

    #csvlocation = None

    def __init__(self):
        self.locations = []
        self.datasets = []
    #    csvlocation = "../../data/"

    def loadCSV(self, category):
        csvlocation = "data/"
        questionfile = open(os.path.join(csvlocation, 'Questions.csv'), 'rb')
        factfile = open(os.path.join(csvlocation, 'Facts.csv'), 'rb')
        placesfile = open(os.path.join(csvlocation,'Places.csv'), 'rb')
        questionreader = csv.DictReader(questionfile, delimiter=',')
        factreader = csv.DictReader(factfile, delimiter=',')
        placesreader = csv.DictReader(placesfile, delimiter=',')
        keys = []
        for row in questionreader:
            if row['CategoryDescription'] == category:
                self.datasets.append({"key" : row['FactKey'], "question" : row ['Question']})
                keys.append(row['FactKey'])
        i = 0
        #currently useing name as id :/
        locs = []
        for row in placesreader:
            self.locations.append({"id" : i, "name" : row['Location'], "geometry" : { "type" : "Point", "coordinates" : [row['Lon'], row['Lat']]}, "values" : {} })
            locs.append(row['Location'])
            i = i+1
        for row in factreader:
            if row['Key'] in keys and row['Location'] in locs:
                #have to loop due to data layout
                for loc in self.locations:
                    if loc['name'] == row['Location']:
                        loc.values[row['Key']] = row['Value']
                        break

<commit_msg>Missing commit from Kelvin because of -X<commit_after>from datasource import datasource
import csv
import os

class csvdatasource(datasource):

    #csvlocation = None

    def __init__(self):
        datasource.__init__(self)
    #    csvlocation = "../../data/"

    def loadCSV(self, category):
        csvlocation = "data/"
        questionfile = open(os.path.join(csvlocation, 'Questions.csv'), 'rb')
        factfile = open(os.path.join(csvlocation, 'Facts.csv'), 'rb')
        placesfile = open(os.path.join(csvlocation,'Places.csv'), 'rb')
        questionreader = csv.DictReader(questionfile, delimiter=',')
        factreader = csv.DictReader(factfile, delimiter=',')
        placesreader = csv.DictReader(placesfile, delimiter=',')
        keys = []
        for row in questionreader:
            if row['CategoryDescription'] == category:
                self.datasets.append({"key" : row['FactKey'], "question" : row ['Question']})
                keys.append(row['FactKey'])
        i = 0
        #currently useing name as id :/
        locs = []
        for row in placesreader:
            self.locations.append({"id" : i, "name" : row['Location'], "geometry" : { "type" : "Point", "coordinates" : [row['Lon'], row['Lat']]}, "values" : {} })
            locs.append(row['Location'])
            i = i+1
        for row in factreader:
            if row['Key'] in keys and row['Location'] in locs:
                #have to loop due to data layout
                for loc in self.locations:
                    if loc['name'] == row['Location']:
                        loc['values'][row['Key']] = row['Value']
                        break


{'fraction_non_alphanumeric': 0.14986376021798364, 'fraction_numerical': 0.0012110202845897668, 'mean_word_length': 2.1861137897782066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5377161', 'n_tokens_mistral': 983, 'n_tokens_neox': 928, 'n_words': 229}