<commit_before>import os
import yaml

from utils import slugify


class File(object):

    def __init__(self, path, context):

        self.context = context # site wide config
        self.path = path

        self.filename = os.path.basename(path)
        self.contents, self.config = self.get_contents()
        self.title = self.config['title']

    def get_contents(self):
        """
        Read file contents. Extract yaml config. Return post contents and
        parsed config.
        """
        f = open(self.path, 'r')

        c = ""
        start = False
        end = False
        conf = ""
        for x in f:
            if x.startswith('-'*80):
                if not start:
                    start = True
                else:
                    end = True
                continue

            if start and not end:
                conf += x
            else:
                c += x
        f.close()

        config = yaml.load(conf)
        c = self._process_contents(c)
        return c, config

    def _process_contents(self, text):
        """
        Run contents through a text processor.
        Options:
            - markdown
            - textile
            - restructuredtext
            - html
        """
        p = self.context['text_processor']
        p = p.lower()

        if p == 'markdown':
            from markdown import markdown
            html = markdown(text)
        elif p == 'textile':
            from textile import textile
            html = textile(text)
        elif p == 'restructuredtext':
            from docutils.core import publish_parts
            parts = publish_parts(source=text, writer_name="html4css1")
            html = parts.get('fragment')
        elif p == 'pygmentedmarkdown':
            from pygmentedmarkdown import markdown
            html = markdown(text)
        elif p == 'html':
            html = text
        else:
            raise Exception("Unknown text processor")

        return html


class Post(File):
    """
    Post object. Exposes the following attributes:
        - filename (e.g. 2010-cool-post.md)
        - path (e.g. /home/user/2010-cool-post.md)
        - contents (post content)
        - config (post front config)
        - year, month, date (in the format specified in config.yaml)
        - title
        - url (e.g. /2010/02/cool-post/)
        - categories (list)
        - author (from config.yaml or from front config)

    All attributes are calculated on init.
    """

    def __init__(self, path, context):<commit_msg>Catch import errors when doing text processing.<commit_after>import os
import yaml

from utils import slugify


class File(object):

    def __init__(self, path, context):

        self.context = context # site wide config
        self.path = path

        self.filename = os.path.basename(path)
        self.contents, self.config = self.get_contents()
        self.title = self.config['title']

    def get_contents(self):
        """
        Read file contents. Extract yaml config. Return post contents and
        parsed config.
        """
        f = open(self.path, 'r')

        c = ""
        start = False
        end = False
        conf = ""
        for x in f:
            if x.startswith('-'*80):
                if not start:
                    start = True
                else:
                    end = True
                continue

            if start and not end:
                conf += x
            else:
                c += x
        f.close()

        config = yaml.load(conf)
        c = self._process_contents(c)
        return c, config

    def _process_contents(self, text):
        """
        Run contents through a text processor.
        Options:
            - markdown
            - textile
            - restructuredtext
            - html
        """
        p = self.context['text_processor']
        p = p.lower()

        try:
            if p == 'markdown':
                from markdown import markdown
                html = markdown(text)
            elif p == 'textile':
                from textile import textile
                html = textile(text)
            elif p == 'restructuredtext':
                from docutils.core import publish_parts
                parts = publish_parts(source=text, writer_name="html4css1")
                html = parts.get('fragment')
            elif p == 'pygmentedmarkdown':
                from pygmentedmarkdown import markdown
                html = markdown(text)
            elif p == 'html':
                html = text
            else:
                raise Exception("Unknown text processor")
        except ImportError:
            raise Exception("Unknown text processor.")

        return html


class Post(File):
    """
    Post object. Exposes the following attributes:
        - filename (e.g. 2010-cool-post.md)
        - path (e.g. /home/user/2010-cool-post.md)
        - contents (post content)
        - config (post front config)
        - year, month, date (in the format specified in config.yaml)
        - title
        - url (e.g. /2010/02/cool-post/)
        - categories (list)
        - author (from config.yaml or from front config)

    All attributes are calculated on init.
    """

    def __init__(self, path, context):
{'fraction_non_alphanumeric': 0.0927327781983346, 'fraction_numerical': 0.006813020439061317, 'mean_word_length': 1.4332412523020257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3173549', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1348, 'n_words': 424}