<commit_before># -*- coding: utf-8 -*-
"""
-------------------------
volt.engine.builtins.blog
-------------------------

Volt Blog Engine.

The blog engine takes text files as resources and writes the static files
constituting a simple blog.

:copyright: (c) 2012 <NAME> <<EMAIL>>
:license: BSD

"""

from __future__ import with_statement
import glob
import os

from volt.config import Config
from volt.engine.core import _RE_DELIM, Engine, Unit
from volt.utils import cachedproperty


ENGINE = 'Text'


class TextUnit(Unit):

    """Class representation of text resources.

    This unit represents resources whose metadata (header) is contained
    in the same file as the content. Some examples of resources like this 
    is a single blog post or a single plain page. 

    """
    def __init__(self, fname, config):
        """Initializes TextUnit.

        fname -- Absolute path to the source file.
        config -- Config object containing unit options.

        """
        super(TextUnit, self).__init__(config)
        self._id = fname

        # parse header and content and check fields
        self.parse_source(self.id)
        self.check_required(self.config.REQUIRED)

        self.logger.debug('created: %s' % self.id)

    @property
    def id(self):
        return self._id

    def parse_source(self, file_path):
        """Parses the header and content from the source file.

        file_path -- Absolute path to source text file.

        """
        with self.open_text(file_path) as source:
            # open file and remove whitespaces
            read = filter(None, _RE_DELIM.split(source.read(), 2))
            # header should be parsed into dict
            self.parse_header(read.pop(0))
            # content is everything else after header
            self.content = read.pop(0).strip()

        # if slug is not set in header, set it now
        if not hasattr(self, 'slug'):
            self.slug = self.slugify(self.title)

        # and set default values
        for field in self.config.DEFAULT_FIELDS:
            if not hasattr(self, field):
                setattr(self, field, self.config.DEFAULT_FIELDS[field])


    def __repr__(self):
        return '%s(%s)' % (type(self).__name__, os.path.basename(self.id))


class TextEngine(Engine):

    """Engine class for processing text units."""

    # Default configurations for the base text engine
    DEFAULTS = Config(
        # Unit filename pattern to match
        UNIT_FNAME_PATTERN = '*'
    )

    @cachedproperty
    def units(self):
        """Units whose source are text files in the filesystem."""
        target_dir = self.config.CONTENT_DIR
        pattern = self.config.UNIT_FNAME_PATTERN
        units = []
        for curdir, children, files in os.walk(target_dir):
            targets = glob.iglob(os.path.join(curdir, pattern))
            files = (x for x in targets if os.path.isfile(x))
            for fname in files:
                units.append(TextUnit(os.path.join(curdir, fname), self.config))

        if units:
            self.logger.debug('created: %s %s' % (len(units), type(units[0]).__name__))
        return units
<commit_msg>Store overridable settings as class attributes<commit_after># -*- coding: utf-8 -*-
"""
-------------------------
volt.engine.builtins.blog
-------------------------

Volt Blog Engine.

The blog engine takes text files as resources and writes the static files
constituting a simple blog.

:copyright: (c) 2012 <NAME> <<EMAIL>>
:license: BSD

"""

from __future__ import with_statement
import glob
import os

from volt.config import Config
from volt.engine.core import _RE_DELIM, Engine, Unit
from volt.utils import cachedproperty


ENGINE = 'Text'


class TextUnit(Unit):

    """Class representation of text resources.

    This unit represents resources whose metadata (header) is contained
    in the same file as the content. Some examples of resources like this 
    is a single blog post or a single plain page. 

    """

    _re_delim = _RE_DELIM

    def __init__(self, fname, config):
        """Initializes TextUnit.

        fname -- Absolute path to the source file.
        config -- Config object containing unit options.

        """
        super(TextUnit, self).__init__(config)
        self._id = fname

        # parse header and content and check fields
        self.parse_source(self.id)
        self.check_required(self.config.REQUIRED)

        self.logger.debug('created: %s' % self.id)

    @property
    def id(self):
        return self._id

    def parse_source(self, file_path):
        """Parses the header and content from the source file.

        file_path -- Absolute path to source text file.

        """
        with self.open_text(file_path) as source:
            # open file and remove whitespaces
            read = filter(None, self._re_delim.split(source.read(), 2))
            # header should be parsed into dict
            self.parse_header(read.pop(0))
            # content is everything else after header
            self.content = read.pop(0).strip()

        # if slug is not set in header, set it now
        if not hasattr(self, 'slug'):
            self.slug = self.slugify(self.title)

        # and set default values
        for field in self.config.DEFAULT_FIELDS:
            if not hasattr(self, field):
                setattr(self, field, self.config.DEFAULT_FIELDS[field])

    def __repr__(self):
        return '%s(%s)' % (type(self).__name__, os.path.basename(self.id))


class TextEngine(Engine):

    """Engine class for processing text units."""

    # Default configurations for the base text engine
    DEFAULTS = Config(
        # Unit filename pattern to match
        UNIT_FNAME_PATTERN = '*'
    )

    unit_class = TextUnit

    @cachedproperty
    def units(self):
        """Units whose source are text files in the filesystem."""
        target_dir = self.config.CONTENT_DIR
        pattern = self.config.UNIT_FNAME_PATTERN
        units = []
        for curdir, children, files in os.walk(target_dir):
            targets = glob.iglob(os.path.join(curdir, pattern))
            files = (x for x in targets if os.path.isfile(x))
            for fname in files:
                units.append(self.unit_class(os.path.join(curdir, fname), self.config))

        if units:
            self.logger.debug('created: %s %s' % (len(units), type(units[0]).__name__))
        return units

{'fraction_non_alphanumeric': 0.11002025868786038, 'fraction_numerical': 0.002805049088359046, 'mean_word_length': 2.2980472764645428, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5873547', 'n_tokens_mistral': 1843, 'n_tokens_neox': 1762, 'n_words': 591}