<commit_before>#!/usr/bin/env python
from __future__ import print_function
from __future__ import division
import os
from contextlib import contextmanager
import imp
import time
import subprocess
import threading
import argh
import shutil
import tempfile
import cw.client
from sqlite3dbm import sshelve
try:
    import cPickle as pickle
except ImportError:
    import pickle

APPS = ['streamcluster', 'blackscholes', 'sobel']
APPSDIR = 'apps'
EVALSCRIPT = 'eval.py'
CONFIGFILE = 'accept_config.txt'
DESCFILE = 'accept_config_desc.txt'
TIMEOUT_FACTOR = 3
MAX_ERROR = 0.3
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

@contextmanager
def chdir(d):
    """A context manager that temporary changes the working directory.
    """
    olddir = os.getcwd()
    os.chdir(d)
    yield
    os.chdir(olddir)

@contextmanager
def sandbox():
    """Create a temporary sandbox directory, copy everything from the
    current directory into it, and enter the directory. Afterwards,
    change back and clean up.
    """
    basedir = tempfile.mkdtemp()
    sandbox_dir = os.path.join(basedir, os.path.basename(os.getcwd()))
    shutil.copytree(os.getcwd(), sandbox_dir)
    with chdir(sandbox_dir):
        yield
    shutil.rmtree(basedir)

class CWMemo(object):
    def __init__(self, dbname='memo.db', host='localhost'):
        self.dbname = dbname
        self.db = sshelve.open(self.dbname)
        self.completion_thread_db = None
        self.client = cw.client.ClientThread(self.completion, host)
        self.jobs = {}
        self.completion_cond = self.client.jobs_cond

    def __enter__(self):
        self.client.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.client.wait()
        self.client.stop()
        self.db.close()

    def completion(self, jobid, output):
        # Called in a different thread, so we need a separate SQLite
        # connection.
        if not self.completion_thread_db:
            self.completion_thread_db = sshelve.open(self.dbname)
        db = self.completion_thread_db

        # Get the arguments and save them with the result.
        with self.client.jobs_cond:
            key = self.jobs.pop(jobid)
        db[key] = output<commit_msg>don't wait after crash<commit_after>#!/usr/bin/env python
from __future__ import print_function
from __future__ import division
import os
from contextlib import contextmanager
import imp
import time
import subprocess
import threading
import argh
import shutil
import tempfile
import cw.client
from sqlite3dbm import sshelve
try:
    import cPickle as pickle
except ImportError:
    import pickle

APPS = ['streamcluster', 'blackscholes', 'sobel']
APPSDIR = 'apps'
EVALSCRIPT = 'eval.py'
CONFIGFILE = 'accept_config.txt'
DESCFILE = 'accept_config_desc.txt'
TIMEOUT_FACTOR = 3
MAX_ERROR = 0.3
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

@contextmanager
def chdir(d):
    """A context manager that temporary changes the working directory.
    """
    olddir = os.getcwd()
    os.chdir(d)
    yield
    os.chdir(olddir)

@contextmanager
def sandbox():
    """Create a temporary sandbox directory, copy everything from the
    current directory into it, and enter the directory. Afterwards,
    change back and clean up.
    """
    basedir = tempfile.mkdtemp()
    sandbox_dir = os.path.join(basedir, os.path.basename(os.getcwd()))
    shutil.copytree(os.getcwd(), sandbox_dir)
    with chdir(sandbox_dir):
        yield
    shutil.rmtree(basedir)

class CWMemo(object):
    def __init__(self, dbname='memo.db', host='localhost'):
        self.dbname = dbname
        self.db = sshelve.open(self.dbname)
        self.completion_thread_db = None
        self.client = cw.client.ClientThread(self.completion, host)
        self.jobs = {}
        self.completion_cond = self.client.jobs_cond

    def __enter__(self):
        self.client.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if not exc_type:
            self.client.wait()
        self.client.stop()
        self.db.close()

    def completion(self, jobid, output):
        # Called in a different thread, so we need a separate SQLite
        # connection.
        if not self.completion_thread_db:
            self.completion_thread_db = sshelve.open(self.dbname)
        db = self.completion_thread_db

        # Get the arguments and save them with the result.
        with self.client.jobs_cond:
            key = self.jobs.pop(jobid)
        db[key] = output
{'fraction_non_alphanumeric': 0.0960352422907489, 'fraction_numerical': 0.001762114537444934, 'mean_word_length': 3.116953762466002, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6155752', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1412, 'n_words': 377}