<commit_before>'''
Created on Mar 12, 2013

@author: Max
'''
import os
import sys
import csv
import itertools
from gptwosample.twosample.twosample_base import TwoSampleBase
import pickle

root = sys.argv[1]
if not os.path.exists(root):
    sys.exit(1)

s = "writing back bayes factors... "
twosample = TwoSampleBase()

def write_bayes_factors(writer, gt_names, likelihoods):
    for name, l in itertools.izip(gt_names, likelihoods):
        writer.writerow([name, twosample.bayes_factor(l)])

for parent, folders, files in os.walk(root):
    bayes_file_name = os.path.join(root, 'bayes.csv')
    if "jobs" == os.path.basename(parent):
        with open(os.path.join(os.path.dirname(parent), "bayes.csv"), 'w') as bayesfile:
            writer = csv.writer(bayesfile)
            N = int(os.path.splitext(files[0])[0].split("_")[-1])
            for Ni in range(N):
                try:
                    sys.stdout.flush()
                    sys.stdout.write("{}/{}        \r".format(Ni,N))
                    with open(os.path.join(parent,'likelihoods_job_{}_{}.pickle'.format(Ni, N)),'r') as liks, \
                         open(os.path.join(parent,'gt_names_job_{}_{}.pickle'.format(Ni, N)),'r') as gts:
                        write_bayes_factors(writer, pickle.load(gts), pickle.load(liks))
                except IOError as io:
                    pass
                    #print io.message
                <commit_msg>collect handling if no files found<commit_after>'''
Created on Mar 12, 2013

@author: Max
'''
import os
import sys
import csv
import itertools
from gptwosample.twosample.twosample_base import TwoSampleBase
import pickle

root = sys.argv[1]
if not os.path.exists(root):
    sys.exit(1)

s = "writing back bayes factors... "
twosample = TwoSampleBase()

def write_bayes_factors(writer, gt_names, likelihoods):
    for name, l in itertools.izip(gt_names, likelihoods):
        writer.writerow([name, twosample.bayes_factor(l)])

for parent, folders, files in os.walk(root):
    bayes_file_name = os.path.join(root, 'bayes.csv')
    if "jobs" == os.path.basename(parent):
        with open(os.path.join(os.path.dirname(parent), "bayes.csv"), 'w') as bayesfile:
            writer = csv.writer(bayesfile)
            try:
                N = int(os.path.splitext(files[0])[0].split("_")[-1])
            except IndexError:
                continue
            for Ni in range(N):
                try:
                    sys.stdout.flush()
                    sys.stdout.write("{}/{}        \r".format(Ni,N))
                    with open(os.path.join(parent,'likelihoods_job_{}_{}.pickle'.format(Ni, N)),'r') as liks, \
                         open(os.path.join(parent,'gt_names_job_{}_{}.pickle'.format(Ni, N)),'r') as gts:
                        write_bayes_factors(writer, pickle.load(gts), pickle.load(liks))
                except IOError as io:
                    pass
                    #print io.message
                
{'fraction_non_alphanumeric': 0.13725490196078433, 'fraction_numerical': 0.0074374577417173765, 'mean_word_length': 2.237417943107221, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7059006', 'n_tokens_mistral': 1008, 'n_tokens_neox': 948, 'n_words': 201}