# Path: streamcorpus_pipeline/stages.py
# class BatchTransform(Configured):
#     '''Transform that acts on an entire :class:`streamcorpus.Chunk` file.'''
#     __metaclass__ = ABCMeta
# 
#     @abstractmethod
#     def process_path(self, chunk_path):
#         '''Process a chunk file at a specific path.
# 
#         Implementations should work in place, writing results out to the
#         same `chunk_path`.  The pipeline guarantees that this stage
#         will have a key ``tmp_dir_path`` in its configuration, and this
#         can be used to store an intermediate file which is then renamed
#         over `chunk_path`.
#         '''
#         raise NotImplementedError('BatchTransform.process_path not implemented')
# 
#     @abstractmethod
#     def shutdown(self):
#         '''Do an orderly shutdown of this stage.
# 
#         If the stage spawns a child process, kill it, and do any other
#         required cleanup.  :meth:`streamcorpus_pipeline.Pipeline._cleanup`
#         will call this method on every batch transform stage, regardless
#         of whether this is the currently running stage or not.
#         '''
#         raise NotImplementedError('shutdown is a required method for all BatchTransforms')
# 
# Path: streamcorpus_pipeline/_taggers.py
# def make_memory_info_msg(clean_visible_path=None, ner_xml_path=None):
#     msg = 'reporting memory while running on:\n%r\n%r' % (clean_visible_path, ner_xml_path)
#     msg += 'VmSize: %d bytes' % _memory.memory()
#     msg += 'VmRSS:  %d bytes' % _memory.resident()
#     msg += 'VmStk:  %d bytes' % _memory.stacksize()
#     msg += 'uncollectable garbage: %r' % gc.garbage
#     msg += 'gc.get_count() = %r' % repr(gc.get_count())
#     ## Do not do gc.get_objects, because it causes nltk to
#     ## load stuff that is not actually in memory yet!
#     #msg += 'current objects: %r' % gc.get_objects()
#     return msg
# 
# def align_labels(t_path1, config):
#     if not ('align_labels_by' in config and config['align_labels_by']):
#         return
#     assert 'aligner_data' in config, 'config missing "aligner_data"'
#     aligner = AlignmentStrategies[ config['align_labels_by'] ]
#     _aligner_core(t_path1, aligner, config['aligner_data'])
# 
# Path: streamcorpus_pipeline/_exceptions.py
# class PipelineOutOfMemory(PipelineBaseException):
#     pass
# 
# class PipelineBaseException(Exception):
#     pass


import gc
import logging
import os
import shutil
import subprocess
import time
import traceback
import uuid
from yakonfig import ConfigurationError
from streamcorpus_pipeline.stages import BatchTransform
from streamcorpus_pipeline._taggers import make_memory_info_msg, align_labels
from streamcorpus_pipeline._exceptions import PipelineOutOfMemory, \
    PipelineBaseException
        tmp_chunk_path = os.path.join(tmp_dir, 'output',
                                      os.path.basename(chunk_path))

        cmd = [
            os.path.join(self.tagger_root_path, self.config['serif_exe']),
            par_path,
            '-o', tmp_dir,
            chunk_path,
        ]

        logger.info('serif cmd: %r', cmd)

        start_time = time.time()
        # make sure we are using as little memory as possible
        gc.collect()
        try:
            self._child = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                                           shell=False)
        except OSError, exc:
            logger.error('error running serif cmd %r', cmd, exc_info=True)
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg()
            logger.critical(msg)
            raise

        s_out, errors = self._child.communicate()

        if not self._child.returncode == 0:
            if self._child.returncode == 137:
                msg = 'tagger returncode = 137\n' + errors

# Path: streamcorpus_pipeline/stages.py
# class BatchTransform(Configured):
#     '''Transform that acts on an entire :class:`streamcorpus.Chunk` file.'''
#     __metaclass__ = ABCMeta
# 
#     @abstractmethod
#     def process_path(self, chunk_path):
#         '''Process a chunk file at a specific path.
# 
#         Implementations should work in place, writing results out to the
#         same `chunk_path`.  The pipeline guarantees that this stage
#         will have a key ``tmp_dir_path`` in its configuration, and this
#         can be used to store an intermediate file which is then renamed
#         over `chunk_path`.
#         '''
#         raise NotImplementedError('BatchTransform.process_path not implemented')
# 
#     @abstractmethod
#     def shutdown(self):
#         '''Do an orderly shutdown of this stage.
# 
#         If the stage spawns a child process, kill it, and do any other
#         required cleanup.  :meth:`streamcorpus_pipeline.Pipeline._cleanup`
#         will call this method on every batch transform stage, regardless
#         of whether this is the currently running stage or not.
#         '''
#         raise NotImplementedError('shutdown is a required method for all BatchTransforms')
# 
# Path: streamcorpus_pipeline/_taggers.py
# def make_memory_info_msg(clean_visible_path=None, ner_xml_path=None):
#     msg = 'reporting memory while running on:\n%r\n%r' % (clean_visible_path, ner_xml_path)
#     msg += 'VmSize: %d bytes' % _memory.memory()
#     msg += 'VmRSS:  %d bytes' % _memory.resident()
#     msg += 'VmStk:  %d bytes' % _memory.stacksize()
#     msg += 'uncollectable garbage: %r' % gc.garbage
#     msg += 'gc.get_count() = %r' % repr(gc.get_count())
#     ## Do not do gc.get_objects, because it causes nltk to
#     ## load stuff that is not actually in memory yet!
#     #msg += 'current objects: %r' % gc.get_objects()
#     return msg
# 
# def align_labels(t_path1, config):
#     if not ('align_labels_by' in config and config['align_labels_by']):
#         return
#     assert 'aligner_data' in config, 'config missing "aligner_data"'
#     aligner = AlignmentStrategies[ config['align_labels_by'] ]
#     _aligner_core(t_path1, aligner, config['aligner_data'])
# 
# Path: streamcorpus_pipeline/_exceptions.py
# class PipelineOutOfMemory(PipelineBaseException):
#     pass
# 
# class PipelineBaseException(Exception):
#     pass


# Path: streamcorpus_pipeline/_serif.py
import gc
import logging
import os
import shutil
import subprocess
import time
import traceback
import uuid
from yakonfig import ConfigurationError
from streamcorpus_pipeline.stages import BatchTransform
from streamcorpus_pipeline._taggers import make_memory_info_msg, align_labels
from streamcorpus_pipeline._exceptions import PipelineOutOfMemory, \
    PipelineBaseException

        tmp_chunk_path = os.path.join(tmp_dir, 'output',
                                      os.path.basename(chunk_path))

        cmd = [
            os.path.join(self.tagger_root_path, self.config['serif_exe']),
            par_path,
            '-o', tmp_dir,
            chunk_path,
        ]

        logger.info('serif cmd: %r', cmd)

        start_time = time.time()
        # make sure we are using as little memory as possible
        gc.collect()
        try:
            self._child = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                                           shell=False)
        except OSError, exc:
            logger.error('error running serif cmd %r', cmd, exc_info=True)
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg()
            logger.critical(msg)
            raise

        s_out, errors = self._child.communicate()

        if not self._child.returncode == 0:
            if self._child.returncode == 137:
                msg = 'tagger returncode = 137\n' + errors

                msg += make_memory_info_msg()