<commit_before>import glob, csv, re, shutil
import numpy as np

oddsfile = list(sorted(glob.glob('raw/odds*.csv')))[-1]
timestamp = re.search('s(.*?)\.', oddsfile).group(1)
with open(oddsfile) as infile:
    reader = csv.reader(infile)
    header = reader.next()
    teams = [row for row in reader]

fixed = []
for team in teams:
    t = team[0:2]
    for odd in team[2:]:
        if odd:
            o = float(odd)
            # betdaq lists some impossible odds. WTF?
            if o < 1: o = 1.
            t.append(o)
    fixed.append(t)

teams = fixed

summary = []
for team in teams:
    odds = team[2:]
    max_ = max(odds)
    min_ = min(odds)
    mean = np.mean(odds)
    median = np.median(odds)
    summary.append(team[:2] + [max_, min_, mean, median])

summaryfile = "raw/summary%s.csv" % timestamp
with file(summaryfile, 'w') as outfile:
    w = csv.writer(outfile)
    w.writerow(['name', 'group', 'max', 'min', 'mean', 'median'])<commit_msg>write 25th and 75th pctiles<commit_after>import glob, csv, re, shutil
import numpy as np

oddsfile = list(sorted(glob.glob('raw/odds*.csv')))[-1]
timestamp = re.search('s(.*?)\.', oddsfile).group(1)
with open(oddsfile) as infile:
    reader = csv.reader(infile)
    header = reader.next()
    teams = [row for row in reader]

fixed = []
for team in teams:
    t = team[0:2]
    for odd in team[2:]:
        if odd:
            o = float(odd)
            # betdaq lists some impossible odds. WTF?
            if o < 1: o = 1.
            t.append(o)
    fixed.append(t)

teams = fixed

summary = []
for team in teams:
    odds = team[2:]
    max_ = max(odds)
    min_ = min(odds)
    mean = np.mean(odds)
    median = np.median(odds)
    twentyfive = np.percentile(odds, 25)
    seventyfive = np.percentile(odds, 75)
    summary.append(team[:2] + [max_, min_, mean, median, twentyfive, seventyfive])

summaryfile = "raw/summary%s.csv" % timestamp
with file(summaryfile, 'w') as outfile:
    w = csv.writer(outfile)
    w.writerow(['name', 'group', 'max', 'min', 'mean', 'median', 'twentyfive', 'seventyfive'])
{'fraction_non_alphanumeric': 0.15980629539951574, 'fraction_numerical': 0.012590799031476998, 'mean_word_length': 2.8330241187384044, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6969472', 'n_tokens_mistral': 774, 'n_tokens_neox': 738, 'n_words': 201}