<commit_before># -*- coding: utf-8 -*-
"""
    app.content
    ~~~~~~~~~~~

    Load raw content for generating pages

    Content comes from folders in the `content` folder. It is expected that
    these subfolders each represent a sort of type of content, with all data
    contained being of the same type.

    One function should be written for each folder, decorated by
    `load(type_name)`. It should return some kind of object representing all
    of the content of that type.

    That object will be keyed by `type_name` into an importable dict in this
    module, called `content`.

    At present, all content is loaded at import time.

"""

import os
import re
import json
import codecs
import unicodedata
from datetime import date
from markdown import markdown, Markdown  # quick-use, extensions-able
from flask import Markup
from app import app


content = {}
content_path = os.path.join(app.root_path, app.config['CONTENT_FOLDER'])


meta_converters = {
    'noop': lambda x: x,
    'iso-date': lambda x: [date(*map(int, t.split('-'))) for t in x],
    'float': lambda x: map(float, x),
    'slugify': lambda x: [s.lowercase().replace(' ', '-') for s in x],
    'one': lambda x: x[0],
}


def slugify(value):
    """Converts to lowerase, removes non-word characters, spaces to hyphens

    Borrowed from django -- http://djangoproject.org -- BSD? licensed
    """
    value = unicodedata.normalize('NFKD', value).\
                        encode('ascii', 'ignore').\
                        decode('ascii')
    value = re.sub('[^\w\s-]', '', value).strip().lower()
    value = re.sub('[-\s]+', '-', value)
    return value


class MetaError(ValueError):
    """Errors arising from reading metadata"""
    def apply_context(self, **context):
        contextualized_message = self.message.format(**context)
        self.__init__(contextualized_message)<commit_msg>be more explicit about date methods<commit_after># -*- coding: utf-8 -*-
"""
    app.content
    ~~~~~~~~~~~

    Load raw content for generating pages

    Content comes from folders in the `content` folder. It is expected that
    these subfolders each represent a sort of type of content, with all data
    contained being of the same type.

    One function should be written for each folder, decorated by
    `load(type_name)`. It should return some kind of object representing all
    of the content of that type.

    That object will be keyed by `type_name` into an importable dict in this
    module, called `content`.

    At present, all content is loaded at import time.

"""

import os
import re
import json
import codecs
import unicodedata
import datetime
from markdown import markdown, Markdown  # quick-use, extensions-able
from flask import Markup
from app import app


content = {}
content_path = os.path.join(app.root_path, app.config['CONTENT_FOLDER'])


meta_converters = {
    'noop': lambda x: x,
    'iso-date': lambda x: [datetime.date(*map(int, t.split('-'))) for t in x],
    'float': lambda x: map(float, x),
    'slugify': lambda x: [s.lowercase().replace(' ', '-') for s in x],
    'one': lambda x: x[0],
}


def slugify(value):
    """Converts to lowerase, removes non-word characters, spaces to hyphens

    Borrowed from django -- http://djangoproject.org -- BSD? licensed
    """
    value = unicodedata.normalize('NFKD', value).\
                        encode('ascii', 'ignore').\
                        decode('ascii')
    value = re.sub('[^\w\s-]', '', value).strip().lower()
    value = re.sub('[-\s]+', '-', value)
    return value


class MetaError(ValueError):
    """Errors arising from reading metadata"""
    def apply_context(self, **context):
        contextualized_message = self.message.format(**context)
        self.__init__(contextualized_message)
{'fraction_non_alphanumeric': 0.12622386874834612, 'fraction_numerical': 0.0010584810796507012, 'mean_word_length': 3.1131664853101197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7506545', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1082, 'n_words': 402}