# Path: streamcorpus_pipeline/_clean_html.py
# def force_unicode(raw):
#     '''Try really really hard to get a Unicode copy of a string.
# 
#     First try :class:`BeautifulSoup.UnicodeDammit` to try to force
#     to Unicode; if that fails, assume UTF-8 encoding, and ignore
#     all errors.
# 
#     :param str raw: string to coerce
#     :return: Unicode approximation of `raw`
#     :returntype: :class:`unicode`
# 
#     '''
#     converted = UnicodeDammit(raw, isHTML=True)
#     if not converted.unicode:
#         converted.unicode = unicode(raw, 'utf8', errors='ignore')
# 
#     encoding_m = encoding_re.match(converted.unicode)
#     if encoding_m:
#         converted.unicode = \
#             encoding_m.group('start_xml') + \
#             encoding_m.group('remainder')
# 
#     return converted.unicode
# 
# Path: streamcorpus_pipeline/_exceptions.py
# class InvalidStreamItem(PipelineBaseException):
#     '''Some content in a stream item was invalid.'''
#     pass
# 
# Path: streamcorpus_pipeline/stages.py
# class Configured(object):
#     '''Any object containing a configuration.
# 
#     The configuration is passed to the constructor.
# 
#     .. attribute:: config
# 
#         The configuration dictionary for this object.
# 
#     '''
#     def __init__(self, config, *args, **kwargs):
#         super(Configured, self).__init__(*args, **kwargs)
#         self.config = config


import sys
from streamcorpus_pipeline._clean_html import force_unicode
from streamcorpus_pipeline._exceptions import InvalidStreamItem
from streamcorpus_pipeline.stages import Configured
'''
kba.pipeline incremental Transform for promoting spinn3r's cleansed
content form (called "content_extract") into the clean_html position
if our own attempt to make clean_html from raw failed.

This software is released under an MIT/X11 open source license.

Copyright 2012-2014 Diffeo, Inc.
'''
from __future__ import absolute_import


class handle_unconvertible_spinn3r(Configured):
    '''
    It seems that some of the spinn3r content is not actually UTF-8
    and there is no record of the original encoding, so we take a shot
    at converting the spinn3r-provided "content_extract" into utf8 and
    using it as the clean_html.  If that fails, we drop the entire
    document from the corpus.
    '''
    config_name = 'handle_unconvertible_spinn3r'

    def __call__(self, si, context):

        if not si.body:
            raise InvalidStreamItem('si.body should never be none: %r' %
                                    si.stream_id)

        if not si.body.raw:

# Path: streamcorpus_pipeline/_clean_html.py
# def force_unicode(raw):
#     '''Try really really hard to get a Unicode copy of a string.
# 
#     First try :class:`BeautifulSoup.UnicodeDammit` to try to force
#     to Unicode; if that fails, assume UTF-8 encoding, and ignore
#     all errors.
# 
#     :param str raw: string to coerce
#     :return: Unicode approximation of `raw`
#     :returntype: :class:`unicode`
# 
#     '''
#     converted = UnicodeDammit(raw, isHTML=True)
#     if not converted.unicode:
#         converted.unicode = unicode(raw, 'utf8', errors='ignore')
# 
#     encoding_m = encoding_re.match(converted.unicode)
#     if encoding_m:
#         converted.unicode = \
#             encoding_m.group('start_xml') + \
#             encoding_m.group('remainder')
# 
#     return converted.unicode
# 
# Path: streamcorpus_pipeline/_exceptions.py
# class InvalidStreamItem(PipelineBaseException):
#     '''Some content in a stream item was invalid.'''
#     pass
# 
# Path: streamcorpus_pipeline/stages.py
# class Configured(object):
#     '''Any object containing a configuration.
# 
#     The configuration is passed to the constructor.
# 
#     .. attribute:: config
# 
#         The configuration dictionary for this object.
# 
#     '''
#     def __init__(self, config, *args, **kwargs):
#         super(Configured, self).__init__(*args, **kwargs)
#         self.config = config


# Path: streamcorpus_pipeline/_handle_unconvertible_spinn3r.py
import sys
from streamcorpus_pipeline._clean_html import force_unicode
from streamcorpus_pipeline._exceptions import InvalidStreamItem
from streamcorpus_pipeline.stages import Configured

'''
kba.pipeline incremental Transform for promoting spinn3r's cleansed
content form (called "content_extract") into the clean_html position
if our own attempt to make clean_html from raw failed.

This software is released under an MIT/X11 open source license.

Copyright 2012-2014 Diffeo, Inc.
'''
from __future__ import absolute_import


class handle_unconvertible_spinn3r(Configured):
    '''
    It seems that some of the spinn3r content is not actually UTF-8
    and there is no record of the original encoding, so we take a shot
    at converting the spinn3r-provided "content_extract" into utf8 and
    using it as the clean_html.  If that fails, we drop the entire
    document from the corpus.
    '''
    config_name = 'handle_unconvertible_spinn3r'

    def __call__(self, si, context):

        if not si.body:
            raise InvalidStreamItem('si.body should never be none: %r' %
                                    si.stream_id)

        if not si.body.raw:

            raise InvalidStreamItem('si.body.raw should never be none: %r' %