<commit_before>#!/usr/bin/env python
# −*− coding: UTF−8 −*−
from __future__ import division

import os

import shelve
import time
import datetime
from datetime import timedelta

from pprint import PrettyPrinter

pprinter = PrettyPrinter()

class Experiment(object):
	def __init__(self, parameters, store_prefix=''):
		self.parameters = parameters
		self.store_prefix = store_prefix

	def get_system(self):
		sys_class = self.parameters['system']
		sys = sys_class(**self.parameters.get('system_params', {}))
		return sys

	def get_solver(self):
		parameters = self.parameters
		sys = self.get_system()
		scheme_class = parameters['scheme']
		scheme = scheme_class(**parameters.get('scheme_params', {}))
		solver_class = parameters['solver']
		solver = solver_class(scheme, sys)
		return solver

	def run(self, save=True):
		parameters = self.parameters
		self.solver = self.get_solver()
		self.solver.initialize(**parameters['initialize'])
		print '-'*80
		print self.get_path()
		print parameters['name']
		pprinter.pprint(parameters)
		start_time = time.time()
		try:
			self.solver.run()
		except Exception as e:
			print e
		finally:
			end_time = time.time()
			self.duration = end_time - start_time
			print str(timedelta(seconds=self.duration))
			self.timestamp = datetime.datetime.now()
			if save:
				self.save()

	def get_path(self):
		path = os.path.join(self.store_prefix, self.parameters['family'])
		return path


	def save(self):
		parameters = self.parameters
		path = self.get_path()
		shelf = shelve.open(path)
		data = {}
		data['timestamp'] = self.timestamp
		data['duration'] = self.duration
		data['params'] = parameters
		data['events'] = self.solver.events_array
		shelf[parameters['name']] = data
		shelf.close()

	@classmethod
	def load(self, family, name):
		shelf = shelve.open(family)
		data = shelf[name]
		params = data['params']
		experiment = self(params)
		solver = experiment.get_solver()
		solver.load_data(data['events'])
		shelf.close()
		return solver

<commit_msg>solver information and solver events are stored in two different slots: it is thus possible to retrieve the event array even if the information is not available because the modules have changed<commit_after>#!/usr/bin/env python
# −*− coding: UTF−8 −*−
from __future__ import division

import os

import shelve
import time
import datetime
from datetime import timedelta
from numpy.lib import format

import StringIO

from pprint import PrettyPrinter

pprinter = PrettyPrinter()

class Experiment(object):
	def __init__(self, parameters, store_prefix=''):
		self.parameters = parameters
		self.store_prefix = store_prefix

	def get_system(self):
		sys_class = self.parameters['system']
		sys = sys_class(**self.parameters.get('system_params', {}))
		return sys

	def get_solver(self):
		parameters = self.parameters
		sys = self.get_system()
		scheme_class = parameters['scheme']
		scheme = scheme_class(**parameters.get('scheme_params', {}))
		solver_class = parameters['solver']
		solver = solver_class(scheme, sys)
		return solver

	def run(self, save=True):
		parameters = self.parameters
		self.solver = self.get_solver()
		self.solver.initialize(**parameters['initialize'])
		print '-'*80
		print self.get_path()
		print parameters['name']
		pprinter.pprint(parameters)
		start_time = time.time()
		try:
			self.solver.run()
		except Exception as e:
			print e
		finally:
			end_time = time.time()
			self.duration = end_time - start_time
			print str(timedelta(seconds=self.duration))
			self.timestamp = datetime.datetime.now()
			if save:
				self.save()

	def get_path(self):
		path = os.path.join(self.store_prefix, self.parameters['family'])
		return path


	def save(self):
		parameters = self.parameters
		path = self.get_path()
		shelf = shelve.open(path)
		name = parameters['name']
		info = {}
		info['timestamp'] = self.timestamp
		info['duration'] = self.duration
		info['params'] = parameters
		event_string = StringIO.StringIO()
		events = self.solver.events_array
		format.write_array(event_string, events)
		shelf[name+'_info'] = info
		shelf[name+'_events'] = event_string.getvalue()
		shelf.close()

	@classmethod
	def load(self, family, name):
		shelf = shelve.open(family)
		info = shelf[name+'_info']
		event_string = shelf[name+'_events']
		event_file = StringIO.StringIO(event_string)
		events = format.read_array(event_file)

		params = info['params']
		experiment = self(params)
		solver = experiment.get_solver()
		solver.load_data(events)
		shelf.close()
		return solver


{'fraction_non_alphanumeric': 0.12411347517730496, 'fraction_numerical': 0.0013297872340425532, 'mean_word_length': 4.61318407960199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5132953', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1445, 'n_words': 345}