<commit_before>import base64, logging
import xml.sax.saxutils
from uuid import uuid4

from pydocx.DocxParser import DocxParser


class Docx2Markdown(DocxParser):

    def __init__(
            self,
            path,         
            convert_root_level_upper_roman=False,
            escape_text=False,
            pseudo_tab_indent=False,
            list_spacing=False,
            process_images=False,
            *args,
            **kwargs):
        self._images = {}
        self.md_tab = "&nbsp;&nbsp;&nbsp;&nbsp;" # 4 space indent
        self.escape_text = escape_text
        self.pseudo_tab_indent = pseudo_tab_indent
        self.list_spacing = list_spacing
        self.process_images = process_images
        super(Docx2Markdown, self).__init__(path, *args, **kwargs)

    @property
    def parsed(self):
        return self._parsed

    def escape(self, text):
        if self.escape_text:
            return xml.sax.saxutils.quoteattr(text)[1:-1]
        return text

    def linebreak(self, pre=None):
        return '  \n'

    def paragraph(self, text, pre=None):
        return text + '  \n'

    def heading(self, text, heading_value):
        print "HEADING: %s"%heading_value
        return text + '  \n'

    def insertion(self, text, author, date):
        pass

    def hyperlink(self, text, href):
        if text == '':
            return ''
        return '[%s](%s "%s")'%(text, href, text)

    def image_handler(self, image_data, filename):
        if self.process_images and image_data.strip():
            extension = filename.split('.')[-1].lower()
            data = base64.b64encode(image_data)

            image_id = int(uuid4().time_mid)
            self._images[image_id] = (extension, data)
            return image_id
        return ''

    # relies on backend to serve images
    def image(self, image_data, filename, x, y):

        if self.process_images:
            src = self.image_handler(image_data, filename)
            if not src:
                return ''
            src = "http://pydocx.appspot.com/img/%s" %src            
            return '![Picture](%s)' % src       
        return ''<commit_msg>nbsp vs plain tab spacing<commit_after>import base64, logging
import xml.sax.saxutils
from uuid import uuid4

from pydocx.DocxParser import DocxParser


class Docx2Markdown(DocxParser):

    def __init__(
            self,
            path,         
            convert_root_level_upper_roman=False,
            escape_text=False,
            pseudo_tab_indent=False,
            list_spacing=False,
            process_images=False,
            *args,
            **kwargs):
        self._images = {}
        self.md_tab = "&nbsp;&nbsp;&nbsp;&nbsp;" if escape_text else "\t" # 4 space indent
        self.escape_text = escape_text
        self.pseudo_tab_indent = pseudo_tab_indent
        self.list_spacing = list_spacing
        self.process_images = process_images
        super(Docx2Markdown, self).__init__(path, *args, **kwargs)

    @property
    def parsed(self):
        return self._parsed

    def escape(self, text):
        if self.escape_text:
            return xml.sax.saxutils.quoteattr(text)[1:-1]
        return text

    def linebreak(self, pre=None):
        return '  \n'

    def paragraph(self, text, pre=None):
        return text + '  \n'

    def heading(self, text, heading_value):
        print "HEADING: %s"%heading_value
        return text + '  \n'

    def insertion(self, text, author, date):
        pass

    def hyperlink(self, text, href):
        if text == '':
            return ''
        return '[%s](%s "%s")'%(text, href, text)

    def image_handler(self, image_data, filename):
        if self.process_images and image_data.strip():
            extension = filename.split('.')[-1].lower()
            data = base64.b64encode(image_data)

            image_id = int(uuid4().time_mid)
            self._images[image_id] = (extension, data)
            return image_id
        return ''

    # relies on backend to serve images
    def image(self, image_data, filename, x, y):

        if self.process_images:
            src = self.image_handler(image_data, filename)
            if not src:
                return ''
            src = "http://pydocx.appspot.com/img/%s" %src                            
            return '![Picture](%s)' % src       
        return ''
{'fraction_non_alphanumeric': 0.10755880337976707, 'fraction_numerical': 0.0063941539164192734, 'mean_word_length': 1.9634641407307172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2467176', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1285, 'n_words': 300}