<commit_before>from main.experiments.SingleChainExperiment import SingleChainExperiment
from main.tools.GitTools import GitTools
from numpy.linalg.linalg import norm
from numpy.ma.core import mean, std, arange, zeros
from pickle import load
import os

class SingleChainAggregator(object):
    def __init__(self, folders):
        self.folders = folders
        
        f = open(folders[0] + SingleChainExperiment.filenames["gitversion"])
        ref_githash = f.readline().strip()
        ref_gitbranch = f.readline().strip()
        f.close()
        
        for folder in folders:
            # assert that all experiments ran under the same git version
            assert(os.path.exists(folder))
            
            f = open(folder + SingleChainExperiment.filenames["gitversion"])
            githash = f.readline().strip()
            gitbranch = f.readline().strip()
            f.close()
            
            # assert that all git hashs are equal
            assert(githash==ref_githash)
            assert(gitbranch==ref_gitbranch)
            
            # this might be false if script was altered after experiment, comment out then
            assert(githash == GitTools.get_hash())
            assert(gitbranch == GitTools.get_branch())
    
    def load_parameter(self, parameter):
        # load number of iterations
        f = open(self.folders[0] + SingleChainExperiment.filenames["parameters"])
        lines = f.readlines()
        f.close()
        parameters = "\n".join(lines)
        idx_a = parameters.find(parameter)
        idx_b = parameters.find(",", idx_a)
        return parameters[idx_a:idx_b]
    
    def load_raw_results(self):
        self.experiments = [None for _ in range(len(self.folders))]
        for i in range(len(self.folders)):
            filename = self.folders[i] + SingleChainExperiment.filenames["output_folder"] + \
                     os.sep + SingleChainExperiment.filenames["output"]
            print "loading", filename
            f = open(filename , "r")
            self.experiments[i] = load(f)
            f.close()
        <commit_msg>removed a debug print<commit_after>from main.experiments.SingleChainExperiment import SingleChainExperiment
from main.tools.GitTools import GitTools
from numpy.linalg.linalg import norm
from numpy.ma.core import mean, std, arange, zeros
from pickle import load
import os

class SingleChainAggregator(object):
    def __init__(self, folders):
        self.folders = folders
        
        f = open(folders[0] + SingleChainExperiment.filenames["gitversion"])
        ref_githash = f.readline().strip()
        ref_gitbranch = f.readline().strip()
        f.close()
        
        for folder in folders:
            # assert that all experiments ran under the same git version
            assert(os.path.exists(folder))
            
            f = open(folder + SingleChainExperiment.filenames["gitversion"])
            githash = f.readline().strip()
            gitbranch = f.readline().strip()
            f.close()
            
            # assert that all git hashs are equal
            assert(githash==ref_githash)
            assert(gitbranch==ref_gitbranch)
            
            # this might be false if script was altered after experiment, comment out then
            assert(githash == GitTools.get_hash())
            assert(gitbranch == GitTools.get_branch())
    
    def load_parameter(self, parameter):
        # load number of iterations
        f = open(self.folders[0] + SingleChainExperiment.filenames["parameters"])
        lines = f.readlines()
        f.close()
        parameters = "\n".join(lines)
        idx_a = parameters.find(parameter)
        idx_b = parameters.find(",", idx_a)
        return parameters[idx_a:idx_b]
    
    def load_raw_results(self):
        self.experiments = [None for _ in range(len(self.folders))]
        for i in range(len(self.folders)):
            filename = self.folders[i] + SingleChainExperiment.filenames["output_folder"] + \
                     os.sep + SingleChainExperiment.filenames["output"]
            f = open(filename , "r")
            self.experiments[i] = load(f)
            f.close()
        
{'fraction_non_alphanumeric': 0.10004775549188157, 'fraction_numerical': 0.0009551098376313276, 'mean_word_length': 2.244771494965143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2545622', 'n_tokens_mistral': 1169, 'n_tokens_neox': 1159, 'n_words': 272}