<commit_before>import logging
import os
import sys
import markdown
import jinja2
import re
import yaml
import json
import codecs
import shutil
import time

def set_jinja_filters(env):
    import jinja_filters
    env.filters['idsafe'] = jinja_filters.idsafe
    return env

def render_theme_template(f, **kwargs):
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(
        [conf.theme_path, conf.default_theme_path]))
    env = set_jinja_filters(env)
    tpl = env.get_template(f)
    return tpl.render(conf=conf, ctrl=ctrl,
        topt=ctrl.theme_options, sopt=ctrl.social_options,
        **kwargs)

def render_text(text, **kwargs):
    env = set_jinja_filters(jinja2.Environment())
    tpl = env.from_string(text)
    return tpl.render(**kwargs)
    
def u_read(fname):
    fh = codecs.open(fname, encoding='utf-8')
    txt = fh.read()
    fh.close()
    return txt

def u_write(fname, data):
    fh = codecs.open(fname, encoding='utf-8', mode='w')
    fh.write(data)
    fh.close()

class Config(object):

    web_prefix = ''
    base_path = None
    source_dir = 'source'
    upload_dir = 'uploads'
    target_dir = 'docroot'
    link_file = 'links.yaml'
    link_json_file = 'links.json'
    theme_dir = 'themes'
    theme = 'default'
    logger = None
    site = {
        'title':'Example Site',
        'description':'Example Site Description',
        'author':'Example Author',
        'keywords':'example, pykwiki'
    }
    page_tpl = 'page.html'
    docroot_index = 'index.html'
    search_tpl = 'search.html'
    pagelist_tpl = 'pagelist.html'
    menu_tpl = 'menu.html'
    source_ext = '.md'
    target_ext = '.html'
    template_ext = '.tpl'
    time_format = '%H:%M:%S'
    timestamp_format = '%Y-%m-%d %H:%M'
    date_format = '%Y-%m-%d'
    index_file = 'idx.json'
    page_data_file = 'pages.json'
    stop_words = ['the','and']
    markdown_exts = [
        'codehilite','toc',
        'pykwiki.ext.tpl',
        'pykwiki.ext.page',
    ]
    page_conf_re = re.compile('\[\[(.*?)\]\]', re.DOTALL)
    blurb_max = 50
    home_page = 'index'

    config_text_properties = [
        'web_prefix',
        'home_page',
        'theme',
        'blurb_max',
        'site',
        'base_path',
        'markdown_exts',
        'time_format',
        'date_format',
        'timestamp_format',
        'source_ext',
    ]

    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)
        if not self.logger.handlers:
            self.logger.addHandler(logging.StreamHandler())

    def __repr__(self):
        return '<Config() base: %s; source: %s; target: %s;>'\
            %(self.base_path, self.source_dir, self.target_dir)

    def check(self):

        if not self.base_path:
            raise Exception('conf.base_path not set!')

        if not os.path.exists(self.base_path): 
            raise Exception('Base path %s does not exist!'%(self.base_path))

        if not self.source_dir:
            raise Exception('conf.source_dir not set!')
    
        if not self.target_dir:
            raise Exception('conf.target_dir not set!')

        if not self.logger or not isinstance(self.logger, logging.Logger):
            raise Exception('conf.logger is invalid!')

        if self.web_prefix.endswith('/'):
            raise Exception('conf.web_prefix cannot end with "/"')

    @property
    def default_theme_path(self):
        return os.path.join(self.base_path, self.theme_dir, 'default')

    @property
    def link_path(self):
        return os.path.join(self.base_path, self.link_file)

    @property
    def link_json_path(self):
        return os.path.join(self.target_path, self.link_json_file)

    @property
    def theme_path(self):
        return os.path.join(self.base_path, self.theme_dir, self.theme)

    @property
    def source_path(self):
        return os.path.join(self.base_path, self.source_dir)

    @property
    def upload_path(self):
        return os.path.join(self.source_path, self.upload_dir)

    @property
    def upload_target_path(self):
        return os.path.join(self.target_path, self.upload_dir)
    
    @property
    def upload_web_path(self):<commit_msg>Theme names are now case insensitive<commit_after>import logging
import os
import sys
import markdown
import jinja2
import re
import yaml
import json
import codecs
import shutil
import time

def set_jinja_filters(env):
    import jinja_filters
    env.filters['idsafe'] = jinja_filters.idsafe
    return env

def render_theme_template(f, **kwargs):
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(
        [conf.theme_path, conf.default_theme_path]))
    env = set_jinja_filters(env)
    tpl = env.get_template(f)
    return tpl.render(conf=conf, ctrl=ctrl,
        topt=ctrl.theme_options, sopt=ctrl.social_options,
        **kwargs)

def render_text(text, **kwargs):
    env = set_jinja_filters(jinja2.Environment())
    tpl = env.from_string(text)
    return tpl.render(**kwargs)
    
def u_read(fname):
    fh = codecs.open(fname, encoding='utf-8')
    txt = fh.read()
    fh.close()
    return txt

def u_write(fname, data):
    fh = codecs.open(fname, encoding='utf-8', mode='w')
    fh.write(data)
    fh.close()

class Config(object):

    web_prefix = ''
    base_path = None
    source_dir = 'source'
    upload_dir = 'uploads'
    target_dir = 'docroot'
    link_file = 'links.yaml'
    link_json_file = 'links.json'
    theme_dir = 'themes'
    theme = 'default'
    logger = None
    site = {
        'title':'Example Site',
        'description':'Example Site Description',
        'author':'Example Author',
        'keywords':'example, pykwiki'
    }
    page_tpl = 'page.html'
    docroot_index = 'index.html'
    search_tpl = 'search.html'
    pagelist_tpl = 'pagelist.html'
    menu_tpl = 'menu.html'
    source_ext = '.md'
    target_ext = '.html'
    template_ext = '.tpl'
    time_format = '%H:%M:%S'
    timestamp_format = '%Y-%m-%d %H:%M'
    date_format = '%Y-%m-%d'
    index_file = 'idx.json'
    page_data_file = 'pages.json'
    stop_words = ['the','and']
    markdown_exts = [
        'codehilite','toc',
        'pykwiki.ext.tpl',
        'pykwiki.ext.page',
    ]
    page_conf_re = re.compile('\[\[(.*?)\]\]', re.DOTALL)
    blurb_max = 50
    home_page = 'index'

    config_text_properties = [
        'web_prefix',
        'home_page',
        'theme',
        'blurb_max',
        'site',
        'base_path',
        'markdown_exts',
        'time_format',
        'date_format',
        'timestamp_format',
        'source_ext',
    ]

    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.DEBUG)
        if not self.logger.handlers:
            self.logger.addHandler(logging.StreamHandler())

    def __repr__(self):
        return '<Config() base: %s; source: %s; target: %s;>'\
            %(self.base_path, self.source_dir, self.target_dir)

    def check(self):

        if not self.base_path:
            raise Exception('conf.base_path not set!')

        if not os.path.exists(self.base_path): 
            raise Exception('Base path %s does not exist!'%(self.base_path))

        if not self.source_dir:
            raise Exception('conf.source_dir not set!')
    
        if not self.target_dir:
            raise Exception('conf.target_dir not set!')

        if not self.logger or not isinstance(self.logger, logging.Logger):
            raise Exception('conf.logger is invalid!')

        if self.web_prefix.endswith('/'):
            raise Exception('conf.web_prefix cannot end with "/"')

    @property
    def default_theme_path(self):
        return os.path.join(self.base_path, self.theme_dir, 'default')

    @property
    def link_path(self):
        return os.path.join(self.base_path, self.link_file)

    @property
    def link_json_path(self):
        return os.path.join(self.target_path, self.link_json_file)

    @property
    def theme_path(self):
        return os.path.join(self.base_path, self.theme_dir, self.theme.lower())

    @property
    def source_path(self):
        return os.path.join(self.base_path, self.source_dir)

    @property
    def upload_path(self):
        return os.path.join(self.source_path, self.upload_dir)

    @property
    def upload_target_path(self):
        return os.path.join(self.target_path, self.upload_dir)
    
    @property
    def upload_web_path(self):
{'fraction_non_alphanumeric': 0.12788906009244994, 'fraction_numerical': 0.001896408676069693, 'mean_word_length': 2.6607375271149674, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3573606', 'n_tokens_mistral': 2881, 'n_tokens_neox': 2795, 'n_words': 562}