# Path: datarobot_batch_scoring/consts.py
# SENTINEL = Batch(-1, 0, None, '', -1)
# 
# class WriterQueueMsg(object):
#     CTX_WARNING = 'CTX_WARNING'
#     CTX_ERROR = 'CTX_ERROR'
#     RESPONSE = 'RESPONSE'
#     SENTINEL = 'SENTINEL'
# 
# class ProgressQueueMsg(object):
#     SHOVEL_DONE = 'SHOVEL_DONE'
#     SHOVEL_ERROR = 'SHOVEL_ERROR'
#     SHOVEL_CSV_ERROR = 'SHOVEL_CSV_ERROR'
#     SHOVEL_PROGRESS = 'SHOVEL_PROGRESS'
#     NETWORK_DONE = 'NETWORK_DONE'
#     NETWORK_ERROR = 'NETWORK_ERROR'
#     NETWORK_PROGRESS = 'NETWORK_PROGRESS'
#     WRITER_DONE = 'WRITER_DONE'
#     WRITER_ERROR = 'WRITER_ERROR'
#     WRITER_PROGRESS = 'WRITER_PROGRESS'
# 
# REPORT_INTERVAL = 5
# 
# Path: datarobot_batch_scoring/utils.py
# def get_rusage():
#     usage = resource.getrusage(resource.RUSAGE_SELF)
#     return {
#         "utime": usage.ru_utime,
#         "stime": usage.ru_stime,
#         "rss": usage.ru_maxrss,
#     }
# 
# Path: datarobot_batch_scoring/exceptions.py
# class ShelveError(Exception):
#     pass
# 
# class UnexpectedKeptColumnCount(Exception):
#     pass
# 
# class NoPredictionThresholdInResult(Exception):
#     pass


import csv
import glob
import hashlib
import multiprocessing
import operator
import os
import shelve
import signal
import sys
import six
    import dbm.dumb as dumb_dbm
    import dumbdbm as dumb_dbm
from functools import reduce
from time import time
from six.moves import queue
from datarobot_batch_scoring.consts import SENTINEL, \
    WriterQueueMsg, ProgressQueueMsg, REPORT_INTERVAL
from datarobot_batch_scoring.utils import get_rusage
from datarobot_batch_scoring.exceptions import ShelveError, \
    UnexpectedKeptColumnCount, NoPredictionThresholdInResult
                            pred_name=self.ctx.pred_name,
                            pred_threshold_name=self.ctx.pred_threshold_name,
                            pred_decision_name=self.ctx.pred_decision_name,
                            keep_cols=self.ctx.keep_cols,
                            skip_row_id=self.ctx.skip_row_id,
                            fast_mode=self.ctx.fast_mode,
                            delimiter=self.ctx.dialect.delimiter,
                            max_prediction_explanations=(
                                self.ctx.max_prediction_explanations
                            )
                            )
                    except UnexpectedKeptColumnCount:
                        self._ui.fatal('Unexpected number of kept columns ' +
                                       'retrieved. This can happen in ' +
                                       '--fast mode with --keep_cols where ' +
                                       'some cells contain quoted delimiters')
                    except NoPredictionThresholdInResult:
                        self._ui.fatal('No predictionThreshold returned from '
                                       'API. --pred_threshold should be used '
                                       'only for binary classification '
                                       'predictions')
                    except Exception as e:
                        self._ui.fatal(e)

                    self.ctx.checkpoint_batch(batch, written_fields, comb)
                    written += 1
                    rows_done += batch.rows

                    if time() - last_report > REPORT_INTERVAL:
                        self.progress_queue.put((

# Path: datarobot_batch_scoring/consts.py
# SENTINEL = Batch(-1, 0, None, '', -1)
# 
# class WriterQueueMsg(object):
#     CTX_WARNING = 'CTX_WARNING'
#     CTX_ERROR = 'CTX_ERROR'
#     RESPONSE = 'RESPONSE'
#     SENTINEL = 'SENTINEL'
# 
# class ProgressQueueMsg(object):
#     SHOVEL_DONE = 'SHOVEL_DONE'
#     SHOVEL_ERROR = 'SHOVEL_ERROR'
#     SHOVEL_CSV_ERROR = 'SHOVEL_CSV_ERROR'
#     SHOVEL_PROGRESS = 'SHOVEL_PROGRESS'
#     NETWORK_DONE = 'NETWORK_DONE'
#     NETWORK_ERROR = 'NETWORK_ERROR'
#     NETWORK_PROGRESS = 'NETWORK_PROGRESS'
#     WRITER_DONE = 'WRITER_DONE'
#     WRITER_ERROR = 'WRITER_ERROR'
#     WRITER_PROGRESS = 'WRITER_PROGRESS'
# 
# REPORT_INTERVAL = 5
# 
# Path: datarobot_batch_scoring/utils.py
# def get_rusage():
#     usage = resource.getrusage(resource.RUSAGE_SELF)
#     return {
#         "utime": usage.ru_utime,
#         "stime": usage.ru_stime,
#         "rss": usage.ru_maxrss,
#     }
# 
# Path: datarobot_batch_scoring/exceptions.py
# class ShelveError(Exception):
#     pass
# 
# class UnexpectedKeptColumnCount(Exception):
#     pass
# 
# class NoPredictionThresholdInResult(Exception):
#     pass


# Path: datarobot_batch_scoring/writer.py
import csv
import glob
import hashlib
import multiprocessing
import operator
import os
import shelve
import signal
import sys
import six
    import dbm.dumb as dumb_dbm
    import dumbdbm as dumb_dbm
from functools import reduce
from time import time
from six.moves import queue
from datarobot_batch_scoring.consts import SENTINEL, \
    WriterQueueMsg, ProgressQueueMsg, REPORT_INTERVAL
from datarobot_batch_scoring.utils import get_rusage
from datarobot_batch_scoring.exceptions import ShelveError, \
    UnexpectedKeptColumnCount, NoPredictionThresholdInResult

                            pred_name=self.ctx.pred_name,
                            pred_threshold_name=self.ctx.pred_threshold_name,
                            pred_decision_name=self.ctx.pred_decision_name,
                            keep_cols=self.ctx.keep_cols,
                            skip_row_id=self.ctx.skip_row_id,
                            fast_mode=self.ctx.fast_mode,
                            delimiter=self.ctx.dialect.delimiter,
                            max_prediction_explanations=(
                                self.ctx.max_prediction_explanations
                            )
                            )
                    except UnexpectedKeptColumnCount:
                        self._ui.fatal('Unexpected number of kept columns ' +
                                       'retrieved. This can happen in ' +
                                       '--fast mode with --keep_cols where ' +
                                       'some cells contain quoted delimiters')
                    except NoPredictionThresholdInResult:
                        self._ui.fatal('No predictionThreshold returned from '
                                       'API. --pred_threshold should be used '
                                       'only for binary classification '
                                       'predictions')
                    except Exception as e:
                        self._ui.fatal(e)

                    self.ctx.checkpoint_batch(batch, written_fields, comb)
                    written += 1
                    rows_done += batch.rows

                    if time() - last_report > REPORT_INTERVAL:
                        self.progress_queue.put((

                            ProgressQueueMsg.WRITER_PROGRESS, {