<commit_before>#!/usr/bin/python

from xmlproxy.proxybase import proxyError
from options import OptionProcessor

import utils
import headerparse

from optparse import OptionParser
from tempfile import NamedTemporaryFile

import time
import re
import sys
import os
import types
import subprocess

try:
    from lxml import etree
except ImportError:
    print """
ATTENTION:
You need to install lxml to take full advantage of blogtool capabilities.
Without it, downloading posts is disallowed.  If all you need it to publish
posts to a blog, then this should be fine.

Unfortunately, this annoying warning message will pop-up everytime you run
blogtool.
"""
    raw_input('Press <ENTER> to continue...')
    import xml.etree.cElementTree as etree

try:
    import markdown
    MARKDOWN_PRESENT = True
except ImportError:
    print """
ATTENTION:
In order to publish posts with blogtool, python-markdown is required.  Without
it, all blogtool is capable of is some basic blog interaction for listing posts
or listing/ modifying categories.

Also, this annoying message will be displayed everytime you run blogtool.
"""
    raw_input('Press <ENTER> to continue...')
    MARKDOWN_PRESENT = False

################################################################################
#   
#   Error classes for blogtool
#

################################################################################
#   
#  define base class for errors
class blogtoolError(Exception): 
    # to be overriden by the subclass
    def __init__(self, msg):
        self.message = 'blogtoolError: %s' % msg

    # this is typically all that's done with the __str__ method
    def __str__(self):
        return self.message

################################################################################
#   
#
class blogtoolRetry(Exception):
    pass

################################################################################
'''
    FileProcessor Class

    This class contains methods to process a file for posting to a blog.

'''
class FileProcessor():

    if MARKDOWN_PRESENT:
        md = markdown.Markdown(extensions=['typed_list'])

    EXTENDED_ENTRY_RE = re.compile(r'\n### MORE ###\s*\n')

    ############################################################################
    def __init__(self, **kwargs):
        for key in kwargs:
            setattr(self, key, kwargs[key])

    ############################################################################
    def _getHeaderandPostText(self, linelist):
        # find first blank line so we can split the list
        for line in linelist:
            if line.isspace():
                i = linelist.index(line)
                break

        else:
            raise blogtoolError("""
Post file must have a blank line separating header and post text.
""" )

        if len(linelist[0:i]) == 0:
            raise blogtoolError('''
No header found, aborting.
''')
        elif len(linelist[i+1:]) == 0:
            raise blogtoolError('''
No text for post, aborting.
''')

        return ''.join(linelist[0:i]), ''.join(linelist[i + 1:])

    ############################################################################ 
    def _procPost(self, posttext):
        if not MARKDOWN_PRESENT:
            print "Unable to publish post without python-markdown.  Sorry..."
            sys.exit()

        m = self.EXTENDED_ENTRY_RE.search(posttext)
        if m:
            description = posttext[:m.start()]
            extended = posttext[m.end():]
        else:
            description = posttext
            extended = ''

#        self.md = markdown.Markdown(extensions=['typed_list'])
        html_desc = self._procText(description)
        if extended:
            html_ext = self._procText(extended)
        else:
            html_ext = ''

        del self.md

        return html_desc, html_ext

    ############################################################################ <commit_msg>No longer del markdown object after text processing is complete.<commit_after>#!/usr/bin/python

from xmlproxy.proxybase import proxyError
from options import OptionProcessor

import utils
import headerparse

from optparse import OptionParser
from tempfile import NamedTemporaryFile

import time
import re
import sys
import os
import types
import subprocess

try:
    from lxml import etree
except ImportError:
    print """
ATTENTION:
You need to install lxml to take full advantage of blogtool capabilities.
Without it, downloading posts is disallowed.  If all you need it to publish
posts to a blog, then this should be fine.

Unfortunately, this annoying warning message will pop-up everytime you run
blogtool.
"""
    raw_input('Press <ENTER> to continue...')
    import xml.etree.cElementTree as etree

try:
    import markdown
    MARKDOWN_PRESENT = True
except ImportError:
    print """
ATTENTION:
In order to publish posts with blogtool, python-markdown is required.  Without
it, all blogtool is capable of is some basic blog interaction for listing posts
or listing/ modifying categories.

Also, this annoying message will be displayed everytime you run blogtool.
"""
    raw_input('Press <ENTER> to continue...')
    MARKDOWN_PRESENT = False

################################################################################
#   
#   Error classes for blogtool
#

################################################################################
#   
#  define base class for errors
class blogtoolError(Exception): 
    # to be overriden by the subclass
    def __init__(self, msg):
        self.message = 'blogtoolError: %s' % msg

    # this is typically all that's done with the __str__ method
    def __str__(self):
        return self.message

################################################################################
#   
#
class blogtoolRetry(Exception):
    pass

################################################################################
'''
    FileProcessor Class

    This class contains methods to process a file for posting to a blog.

'''
class FileProcessor():

    if MARKDOWN_PRESENT:
        md = markdown.Markdown(extensions=['typed_list'])

    EXTENDED_ENTRY_RE = re.compile(r'\n### MORE ###\s*\n')

    ############################################################################
    def __init__(self, **kwargs):
        for key in kwargs:
            setattr(self, key, kwargs[key])

    ############################################################################
    def _getHeaderandPostText(self, linelist):
        # find first blank line so we can split the list
        for line in linelist:
            if line.isspace():
                i = linelist.index(line)
                break

        else:
            raise blogtoolError("""
Post file must have a blank line separating header and post text.
""" )

        if len(linelist[0:i]) == 0:
            raise blogtoolError('''
No header found, aborting.
''')
        elif len(linelist[i+1:]) == 0:
            raise blogtoolError('''
No text for post, aborting.
''')

        return ''.join(linelist[0:i]), ''.join(linelist[i + 1:])

    ############################################################################ 
    def _procPost(self, posttext):
        if not MARKDOWN_PRESENT:
            print "Unable to publish post without python-markdown.  Sorry..."
            sys.exit()

        m = self.EXTENDED_ENTRY_RE.search(posttext)
        if m:
            description = posttext[:m.start()]
            extended = posttext[m.end():]
        else:
            description = posttext
            extended = ''

#        self.md = markdown.Markdown(extensions=['typed_list'])
        html_desc = self._procText(description)
        if extended:
            html_ext = self._procText(extended)
        else:
            html_ext = ''

        return html_desc, html_ext

    ############################################################################ 
{'fraction_non_alphanumeric': 0.23133383953429512, 'fraction_numerical': 0.0015186028853454822, 'mean_word_length': 3.2881172002170374, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '592563', 'n_tokens_mistral': 2059, 'n_tokens_neox': 1933, 'n_words': 663}