# Path: datarobot_batch_scoring/consts.py
# SENTINEL = Batch(-1, 0, None, '', -1)
# 
# class WriterQueueMsg(object):
#     CTX_WARNING = 'CTX_WARNING'
#     CTX_ERROR = 'CTX_ERROR'
#     RESPONSE = 'RESPONSE'
#     SENTINEL = 'SENTINEL'
# 
# class ProgressQueueMsg(object):
#     SHOVEL_DONE = 'SHOVEL_DONE'
#     SHOVEL_ERROR = 'SHOVEL_ERROR'
#     SHOVEL_CSV_ERROR = 'SHOVEL_CSV_ERROR'
#     SHOVEL_PROGRESS = 'SHOVEL_PROGRESS'
#     NETWORK_DONE = 'NETWORK_DONE'
#     NETWORK_ERROR = 'NETWORK_ERROR'
#     NETWORK_PROGRESS = 'NETWORK_PROGRESS'
#     WRITER_DONE = 'WRITER_DONE'
#     WRITER_ERROR = 'WRITER_ERROR'
#     WRITER_PROGRESS = 'WRITER_PROGRESS'
# 
# REPORT_INTERVAL = 5
# 
# Path: datarobot_batch_scoring/utils.py
# def get_rusage():
#     usage = resource.getrusage(resource.RUSAGE_SELF)
#     return {
#         "utime": usage.ru_utime,
#         "stime": usage.ru_stime,
#         "rss": usage.ru_maxrss,
#     }
# 
# Path: datarobot_batch_scoring/exceptions.py
# class ShelveError(Exception):
#     pass
# 
# class UnexpectedKeptColumnCount(Exception):
#     pass
# 
# class NoPredictionThresholdInResult(Exception):
#     pass


import csv
import glob
import hashlib
import multiprocessing
import operator
import os
import shelve
import signal
import sys
import six
    import dbm.dumb as dumb_dbm
    import dumbdbm as dumb_dbm
from functools import reduce
from time import time
from six.moves import queue
from datarobot_batch_scoring.consts import SENTINEL, \
    WriterQueueMsg, ProgressQueueMsg, REPORT_INTERVAL
from datarobot_batch_scoring.utils import get_rusage
from datarobot_batch_scoring.exceptions import ShelveError, \
    UnexpectedKeptColumnCount, NoPredictionThresholdInResult

    def batch_generator_args(self):
        """
        returns the arguments needed to set up a BatchGenerator
        In this case it's a fresh start
        """
        already_processed_batches = set()
        args = [self.dataset, self.n_samples, self.n_retry, self.delimiter,
                self._ui, self.fast_mode, self.encoding,
                already_processed_batches]
        return args


class OldRunContext(RunContext):
    """RunContext for a resume run.

    It requires a shelve file and plays back the checkpoint journal.
    Checks if inputs are consistent.

    TODO: add md5sum of dataset otherwise they might
    use a different file for resume.
    """

    def __enter__(self):
        if not self.file_context.exists():
            raise ValueError('Cannot resume a project without {}'
                             .format(self.FILENAME))
        super(OldRunContext, self).__enter__()

        if self.db['n_samples'] != self.n_samples:

# Path: datarobot_batch_scoring/consts.py
# SENTINEL = Batch(-1, 0, None, '', -1)
# 
# class WriterQueueMsg(object):
#     CTX_WARNING = 'CTX_WARNING'
#     CTX_ERROR = 'CTX_ERROR'
#     RESPONSE = 'RESPONSE'
#     SENTINEL = 'SENTINEL'
# 
# class ProgressQueueMsg(object):
#     SHOVEL_DONE = 'SHOVEL_DONE'
#     SHOVEL_ERROR = 'SHOVEL_ERROR'
#     SHOVEL_CSV_ERROR = 'SHOVEL_CSV_ERROR'
#     SHOVEL_PROGRESS = 'SHOVEL_PROGRESS'
#     NETWORK_DONE = 'NETWORK_DONE'
#     NETWORK_ERROR = 'NETWORK_ERROR'
#     NETWORK_PROGRESS = 'NETWORK_PROGRESS'
#     WRITER_DONE = 'WRITER_DONE'
#     WRITER_ERROR = 'WRITER_ERROR'
#     WRITER_PROGRESS = 'WRITER_PROGRESS'
# 
# REPORT_INTERVAL = 5
# 
# Path: datarobot_batch_scoring/utils.py
# def get_rusage():
#     usage = resource.getrusage(resource.RUSAGE_SELF)
#     return {
#         "utime": usage.ru_utime,
#         "stime": usage.ru_stime,
#         "rss": usage.ru_maxrss,
#     }
# 
# Path: datarobot_batch_scoring/exceptions.py
# class ShelveError(Exception):
#     pass
# 
# class UnexpectedKeptColumnCount(Exception):
#     pass
# 
# class NoPredictionThresholdInResult(Exception):
#     pass


# Path: datarobot_batch_scoring/writer.py
import csv
import glob
import hashlib
import multiprocessing
import operator
import os
import shelve
import signal
import sys
import six
    import dbm.dumb as dumb_dbm
    import dumbdbm as dumb_dbm
from functools import reduce
from time import time
from six.moves import queue
from datarobot_batch_scoring.consts import SENTINEL, \
    WriterQueueMsg, ProgressQueueMsg, REPORT_INTERVAL
from datarobot_batch_scoring.utils import get_rusage
from datarobot_batch_scoring.exceptions import ShelveError, \
    UnexpectedKeptColumnCount, NoPredictionThresholdInResult


    def batch_generator_args(self):
        """
        returns the arguments needed to set up a BatchGenerator
        In this case it's a fresh start
        """
        already_processed_batches = set()
        args = [self.dataset, self.n_samples, self.n_retry, self.delimiter,
                self._ui, self.fast_mode, self.encoding,
                already_processed_batches]
        return args


class OldRunContext(RunContext):
    """RunContext for a resume run.

    It requires a shelve file and plays back the checkpoint journal.
    Checks if inputs are consistent.

    TODO: add md5sum of dataset otherwise they might
    use a different file for resume.
    """

    def __enter__(self):
        if not self.file_context.exists():
            raise ValueError('Cannot resume a project without {}'
                             .format(self.FILENAME))
        super(OldRunContext, self).__enter__()

        if self.db['n_samples'] != self.n_samples:

            raise ShelveError('n_samples mismatch: should be {} but was {}'