<commit_before>from __future__ import absolute_import
import logging
from cStringIO import StringIO

from docx import Document

from streamcorpus_pipeline.stages import Configured

logger = logging.getLogger(__name__)

def extract_text(data):
    fp = StringIO(data)
    document = Document(fp)

    result = '\n\n'.join(para.text for para in document.paragraphs)

    fp.close()
    return result

class docx_to_text(Configured):
    '''
    returns a kba.pipeline "transform" function that attempts to
    generate stream_item.body.clean_visible from body.raw
    '''
    config_name = 'docx_to_text'
    def __call__(self, stream_item, context):
        if stream_item.body and stream_item.body.raw \
                and stream_item.body.media_type == 'application/msword':

            logger.debug('converting docx to text for %s',
                         stream_item.stream_id)

            try:
                stream_item.body.clean_html = \
                    extract_text(stream_item.body.raw)
            except Exception as exc:
                logger.exception('failed to convert %s from docx',
                                 stream_item.stream_id)

        return stream_item<commit_msg>catching the exception broken docx file instead of logging a full traceback<commit_after>from __future__ import absolute_import
import logging
from cStringIO import StringIO

from docx import Document
from zipfile import BadZipfile

from streamcorpus_pipeline.stages import Configured

logger = logging.getLogger(__name__)

def extract_text(data):
    fp = StringIO(data)
    document = Document(fp)

    result = '\n\n'.join(para.text for para in document.paragraphs)

    fp.close()
    return result

class docx_to_text(Configured):
    '''
    returns a kba.pipeline "transform" function that attempts to
    generate stream_item.body.clean_visible from body.raw
    '''
    config_name = 'docx_to_text'
    def __call__(self, stream_item, context):
        if stream_item.body and stream_item.body.raw \
                and stream_item.body.media_type == 'application/msword':

            logger.debug('converting docx to text for %s',
                         stream_item.stream_id)

            try:
                stream_item.body.clean_html = \
                    extract_text(stream_item.body.raw)
            except BadZipfile:
                logger.info('BadZipfile: broken docx file: %r',
                            stream_item.abs_url)
            except Exception as exc:
                logger.exception('failed to convert %s from docx',
                                 stream_item.stream_id)

        return stream_item
{'fraction_non_alphanumeric': 0.07291273139403098, 'fraction_numerical': 0.0, 'mean_word_length': 2.3434343434343434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4924293', 'n_tokens_mistral': 739, 'n_tokens_neox': 716, 'n_words': 192}