<commit_before>import sys, os, csv
import SonicScrewdriver as utils

metafile = '/Users/tunder/Dropbox/GenreProject/metadata/unifiedficsample.csv'

id2date = dict()
date2wordcount = dict()
dateset = set()

with open(metafile, encoding = 'utf-8') as f:
    reader = csv.reader(f)
    for row in reader:
        # skip header
        if row[0] == "idcode":
            continue
        code = row[0]
        date = int(row[1])
        dateset.add(date)
        wordcount = int(row[2])

        id2date[code] = date
        utils.addtodict(date, wordcount, date2wordcount)

date2moneycount = dict()

datafile = '/Volumes/TARDIS/work/moneycontext/fifteenwords.tsv'

with open(datafile, encoding = 'utf-8') as f:
    for line in f:
        fields = line.split('\t')
        date = int(fields[1])
        if date not in dateset:
            print("Missing " + str(date))
        category = fields[3]
        if category == "money":
            utils.addtodict(date, 1, date2moneycount)

datelist = list(dateset)
datelist.sort()

outfile = '/Users/tunder/Dropbox/GenreProject/python/piketty/model_filtered_money.csv'

with open(outfile, mode='w', encoding = 'utf-8') as f:
    writer = csv.writer(f)
    row = ['date', 'money', 'total', 'ratio']
    writer.writerow(row)
    for date in datelist:
        strdate = str(date)
        total = str(date2wordcount[date])
        money = str(date2moneycount[date])
        ratio = date2moneycount[date] / date2wordcount[date]
        strratio = ('%.6f' % ratio)
        row = [strdate, money, total, strratio]
        writer.writerow(row)


<commit_msg>Updated code to generate wordcounts for stats grads.<commit_after>import sys, os, csv
import SonicScrewdriver as utils

metafile = '/Users/tunder/Dropbox/GenreProject/metadata/ficsampleimprovedmetadata.csv'

id2date = dict()
date2wordcount = dict()
dateset = set()
id2wordcount = dict()

with open(metafile, encoding = 'utf-8') as f:
    reader = csv.reader(f)
    for row in reader:
        # skip header
        if row[0] == "idcode":
            continue
        code = row[0]
        date = int(row[1])
        dateset.add(date)
        wordcount = int(row[3])

        id2date[code] = date
        utils.addtodict(date, wordcount, date2wordcount)
        id2wordcount[code] = wordcount


date2moneycount = dict()
idnword2moneycount = dict()

datafile = '/Volumes/TARDIS/work/moneycontext/fifteenwords.tsv'

with open(datafile, encoding = 'utf-8') as f:
    for line in f:
        fields = line.split('\t')
        code = fields[0]
        date = int(fields[1])
        if date not in dateset:
            print("Missing " + str(date))
        keyword = fields[2]
        category = fields[3]
        if category == "money":
            utils.addtodict(date, 1, date2moneycount)
            if code in idnword2moneycount:
                if keyword in idnword2moneycount[code]:
                    idnword2moneycount[code][keyword] += 1
                else:
                    idnword2moneycount[code][keyword] = 1
            else:
                idnword2moneycount[code] = dict()
                idnword2moneycount[code][keyword] = 1

datelist = list(dateset)
datelist.sort()

outfile = '/Users/tunder/Dropbox/GenreProject/python/piketty/model_filtered_money.csv'

with open(outfile, mode='w', encoding = 'utf-8') as f:
    writer = csv.writer(f)
    row = ['date', 'money', 'total', 'ratio']
    writer.writerow(row)
    for date in datelist:
        strdate = str(date)
        total = str(date2wordcount[date])
        money = str(date2moneycount[date])
        ratio = date2moneycount[date] / date2wordcount[date]
        strratio = ('%.6f' % ratio)
        row = [strdate, money, total, strratio]
        writer.writerow(row)

outfile = '/Users/tunder/Dropbox/GenreProject/python/piketty/filtered_moneywords_by_vol.csv'

with open(outfile, mode = 'w', encoding = 'utf-8') as f:
    writer = csv.writer(f)
    row = ['idcode', 'word', 'count']
    writer.writerow(row)
    for code, subdict in idnword2moneycount.items():
        for word, count in subdict.items():
            row = [code, word, str(count)]
            writer.writerow(row)



{'fraction_non_alphanumeric': 0.12632594021215043, 'fraction_numerical': 0.013982642237222759, 'mean_word_length': 2.5922077922077924, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '507675', 'n_tokens_mistral': 1424, 'n_tokens_neox': 1366, 'n_words': 309}