<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012 <NAME> y otros.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function, unicode_literals
from collections import defaultdict
from copy import copy
import glob
import gzip
import os
import sys
try:
    from urlparse import urlparse, urlsplit, urljoin
except ImportError:
    from urllib.parse import urlparse, urlsplit, urljoin  # NOQA

import lxml.html
from yapsy.PluginManager import PluginManager
import pytz

if os.getenv('DEBUG'):
    import logging
    logging.basicConfig(level=logging.DEBUG)
else:
    import logging
    logging.basicConfig(level=logging.ERROR)

from .post import Post
from . import utils
from .plugin_categories import (
    Command,
    LateTask,
    PageCompiler,
    Task,
    TemplateSystem,
)

config_changed = utils.config_changed

__all__ = ['Nikola']


class Nikola(object):

    """Class that handles site generation.

    Takes a site config as argument on creation.
    """

    def __init__(self, **config):
        """Setup proper environment for running tasks."""

        self.global_data = {}
        self.posts_per_year = defaultdict(list)
        self.posts_per_tag = defaultdict(list)
        self.timeline = []
        self.pages = []
        self._scanned = False
        if not config:
            self.configured = False
        else:
            self.configured = True

        # This is the default config
        # TODO: fill it
        self.config = {
            'ADD_THIS_BUTTONS': True,
            'ANALYTICS': '',
            'ARCHIVE_PATH': "",
            'ARCHIVE_FILENAME': "archive.html",
            'CACHE_FOLDER': 'cache',
            'COMMENTS_IN_GALLERIES': False,
            'COMMENTS_IN_STORIES': False,
            'CONTENT_FOOTER': '',
            'DATE_FORMAT': '%Y-%m-%d %H:%M',
            'DEFAULT_LANG': "en",
            'DEPLOY_COMMANDS': [],
            'DISQUS_FORUM': 'nikolademo',
            'FAVICONS': {},
            'FILE_METADATA_REGEXP': None,
            'FILES_FOLDERS': {'files': ''},
            'FILTERS': {},
            'GALLERY_PATH': 'galleries',
            'GZIP_FILES': False,
            'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js'),
            'INDEX_DISPLAY_POST_COUNT': 10,
            'INDEX_TEASERS': False,
            'INDEXES_TITLE': "",
            'INDEXES_PAGES': "",
            'INDEX_PATH': '',
            'LICENSE': '',
            'LISTINGS_FOLDER': 'listings',
            'MAX_IMAGE_SIZE': 1280,
            'MATHJAX_CONFIG': '',
            'OUTPUT_FOLDER': 'output',
            'post_compilers': {
                "rest": ('.txt', '.rst'),
                "markdown": ('.md', '.mdown', '.markdown'),
                "textile": ('.textile',),
                "txt2tags": ('.t2t',),
                "bbcode": ('.bb',),
                "wiki": ('.wiki',),
                "ipynb": ('.ipynb',),
                "html": ('.html', '.htm')
            },
            'POST_PAGES': (
                ("posts/*.txt", "posts", "post.tmpl", True),
                ("stories/*.txt", "stories", "story.tmpl", False),
            ),
            'REDIRECTIONS': [],
            'RSS_LINK': None,
            'RSS_PATH': '',
            'RSS_TEASERS': True,
            'SEARCH_FORM': '',
            'SLUG_TAG_PATH': True,
            'STORY_INDEX': False,
            'TAG_PATH': 'categories',
            'TAG_PAGES_ARE_INDEXES': False,
            'THEME': 'site',
            'THUMBNAIL_SIZE': 180,
            'USE_BUNDLES': True,
            'USE_CDN': False,
            'USE_FILENAME_AS_TITLE': True,
            'TIMEZONE': None,
        }

        self.config.update(config)
        self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
                                                      {self.config['DEFAULT_'
                                                      'LANG']: ''})

        self.THEMES = utils.get_theme_chain(self.config['THEME'])

        self.MESSAGES = utils.load_messages(self.THEMES,
                                            self.config['TRANSLATIONS'])

        self.plugin_manager = PluginManager(categories_filter={
            "Command": Command,
            "Task": Task,
            "LateTask": LateTask,
            "TemplateSystem": TemplateSystem,
            "PageCompiler": PageCompiler,
        })
        self.plugin_manager.setPluginInfoExtension('plugin')
        self.plugin_manager.setPluginPlaces([
            str(os.path.join(os.path.dirname(__file__), 'plugins')),
            str(os.path.join(os.getcwd(), 'plugins')),
        ])
        self.plugin_manager.collectPlugins()

        self.commands = {}
        # Activate all command plugins
        for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
            self.plugin_manager.activatePluginByName(pluginInfo.name)
            pluginInfo.plugin_object.set_site(self)
            pluginInfo.plugin_object.short_help = pluginInfo.description
            self.commands[pluginInfo.name] = pluginInfo.plugin_object

        # Activate all task plugins
        for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
            self.plugin_manager.activatePluginByName(pluginInfo.name)
            pluginInfo.plugin_object.set_site(self)

        for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
            self.plugin_manager.activatePluginByName(pluginInfo.name)
            pluginInfo.plugin_object.set_site(self)

        # set global_context for template rendering
        self.GLOBAL_CONTEXT = {
        }

        self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
        self.GLOBAL_CONTEXT['_link'] = self.link
        self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
        self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
        self.GLOBAL_CONTEXT['exists'] = self.file_exists
        self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[
            'SLUG_TAG_PATH']

        self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
            'ADD_THIS_BUTTONS']
        self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
            'INDEX_DISPLAY_POST_COUNT']
        self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
        self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
        self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
        self.GLOBAL_CONTEXT['date_format'] = self.config.get(
            'DATE_FORMAT', '%Y-%m-%d %H:%M')
        self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
        self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
        self.GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL')
        if (self.GLOBAL_CONTEXT['blog_url'] is None and
                self.config.get('BLOG_URL')):
            # TODO: warn if using old options
            # print("WARNING: You should configure SITE_URL instead of BLOG_URL")
            # print("See docs at FIXME put URL")
            self.GLOBAL_CONTEXT['blog_url'] = self.config.get('BLOG_URL')
        else:  # FIXME: temporary until I get rid of BLOG_URL
            self.config['BLOG_URL'] = self.config.get('SITE_URL')
        self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
        self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS')
        self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
        self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
        self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
        self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM')
        self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
            'MATHJAX_CONFIG')
        self.GLOBAL_CONTEXT['content_footer'] = self.config.get(
            'CONTENT_FOOTER')
        self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
        self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
        self.GLOBAL_CONTEXT['sidebar_links'] = self.config.get('SIDEBAR_LINKS')
        self.GLOBAL_CONTEXT['twitter_card'] = self.config.get(
            'TWITTER_CARD', {})

        self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))

        # check if custom css exist and is not empty
        for files_path in list(self.config['FILES_FOLDERS'].keys()):
            custom_css_path = os.path.join(files_path, 'assets/css/custom.css')
            if self.file_exists(custom_css_path, not_empty=True):
                self.GLOBAL_CONTEXT['has_custom_css'] = True
                break
        else:
            self.GLOBAL_CONTEXT['has_custom_css'] = False

        # Load template plugin
        template_sys_name = utils.get_template_engine(self.THEMES)
        pi = self.plugin_manager.getPluginByName(
            template_sys_name, "TemplateSystem")
        if pi is None:
            sys.stderr.write("Error loading {0} template system "
                             "plugin\n".format(template_sys_name))
            sys.exit(1)
        self.template_system = pi.plugin_object
        lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates")
                       for name in self.THEMES]
        self.template_system.set_directories(lookup_dirs,
                                             self.config['CACHE_FOLDER'])

        # Load compiler plugins
        self.compilers = {}
        self.inverse_compilers = {}

        for pluginInfo in self.plugin_manager.getPluginsOfCategory(
                "PageCompiler"):
            self.compilers[pluginInfo.name] = \
                pluginInfo.plugin_object.compile_html

    def get_compiler(self, source_name):
        """Get the correct compiler for a post from `conf.post_compilers`

        To make things easier for users, the mapping in conf.py is
        compiler->[extensions], although this is less convenient for us. The
        majority of this function is reversing that dictionary and error
        checking.
        """
        ext = os.path.splitext(source_name)[1]
        try:
            compile_html = self.inverse_compilers[ext]
        except KeyError:
            # Find the correct compiler for this files extension
            langs = [lang for lang, exts in
                     list(self.config['post_compilers'].items())
                     if ext in exts]
            if len(langs) != 1:
                if len(set(langs)) > 1:
                    exit("Your file extension->compiler definition is"
                         "ambiguous.\nPlease remove one of the file extensions"
                         "from 'post_compilers' in conf.py\n(The error is in"
                         "one of {0})".format(', '.join(langs)))
                elif len(langs) > 1:
                    langs = langs[:1]
                else:
                    exit("post_compilers in conf.py does not tell me how to "
                         "handle '{0}' extensions.".format(ext))

            lang = langs[0]
            compile_html = self.compilers[lang]
            self.inverse_compilers[ext] = compile_html

        return compile_html

    def render_template(self, template_name, output_name, context):
        local_context = {}
        local_context["template_name"] = template_name
        local_context.update(self.GLOBAL_CONTEXT)
        local_context.update(context)
        data = self.template_system.render_template(
            template_name, None, local_context)

        assert isinstance(output_name, bytes)
        assert output_name.startswith(
            self.config["OUTPUT_FOLDER"].encode('utf8'))
        url_part = output_name.decode('utf8')[len(self.config["OUTPUT_FOLDER"])
                                              + 1:]

        # This is to support windows paths
        url_part = "/".join(url_part.split(os.sep))

        src = urljoin(self.config["BLOG_URL"], url_part)

        parsed_src = urlsplit(src)
        src_elems = parsed_src.path.split('/')[1:]

        def replacer(dst):
            # Refuse to replace links that are full URLs.
            dst_url = urlparse(dst)
            if dst_url.netloc:
                if dst_url.scheme == 'link':  # Magic link
                    dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
                                    context['lang'])
                else:
                    return dst

            # Normalize
            dst = urljoin(src, dst)
            # Avoid empty links.
            if src == dst:
                return "#"
            # Check that link can be made relative, otherwise return dest
            parsed_dst = urlsplit(dst)
            if parsed_src[:2] != parsed_dst[:2]:
                return dst

            # Now both paths are on the same site and absolute
            dst_elems = parsed_dst.path.split('/')[1:]

            i = 0
            for (i, s), d in zip(enumerate(src_elems), dst_elems):
                if s != d:
                    break
            # Now i is the longest common prefix
            result = '/'.join(['..'] * (len(src_elems) - i - 1) +
                              dst_elems[i:])

            if not result:
                result = "."

            # Don't forget the fragment (anchor) part of the link
            if parsed_dst.fragment:
                result += "#" + parsed_dst.fragment

            assert result, (src, dst, i, src_elems, dst_elems)

            return result

        try:
            os.makedirs(os.path.dirname(output_name))
        except:
            pass
        doc = lxml.html.document_fromstring(data)
        doc.rewrite_links(replacer)
        data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
        with open(output_name, "wb+") as post_file:
            post_file.write(data)

    def path(self, kind, name, lang, is_link=False):
        """Build the path to a certain kind of page.

        kind is one of:

        * tag_index (name is ignored)
        * tag (and name is the tag name)
        * tag_rss (name is the tag name)
        * archive (and name is the year, or None for the main archive index)
        * index (name is the number in index-number)
        * rss (name is ignored)
        * gallery (name is the gallery name)
        * listing (name is the source code file name)
        * post_path (name is 1st element in a post_pages tuple)

        The returned value is always a path relative to output, like
        "categories/whatever.html"

        If is_link is True, the path is absolute and uses "/" as separator
        (ex: "/archive/index.html").
        If is_link is False, the path is relative to output and uses the
        platform's separator.
        (ex: "archive\\index.html")
        """

        path = []

        if kind == "tag_index":
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['TAG_PATH'], 'index.html'] if _f]
        elif kind == "tag":
            if self.config['SLUG_TAG_PATH']:
                name = utils.slugify(name)
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['TAG_PATH'], name + ".html"] if
                    _f]
        elif kind == "tag_rss":
            if self.config['SLUG_TAG_PATH']:
                name = utils.slugify(name)
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['TAG_PATH'], name + ".xml"] if
                    _f]
        elif kind == "index":
            if name not in [None, 0]:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['INDEX_PATH'],
                                      'index-{0}.html'.format(name)] if _f]
            else:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['INDEX_PATH'], 'index.html']
                        if _f]
        elif kind == "post_path":
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  os.path.dirname(name), "index.html"] if _f]
        elif kind == "rss":
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['RSS_PATH'], 'rss.xml'] if _f]
        elif kind == "archive":
            if name:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['ARCHIVE_PATH'], name,
                                      'index.html'] if _f]
            else:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['ARCHIVE_PATH'],
                                      self.config['ARCHIVE_FILENAME']] if _f]
        elif kind == "gallery":
            path = [_f for _f in [self.config['GALLERY_PATH'], name,
                                  'index.html'] if _f]
        elif kind == "listing":
            path = [_f for _f in [self.config['LISTINGS_FOLDER'], name +
                                  '.html'] if _f]
        if is_link:
            return '/' + ('/'.join(path))
        else:
            return os.path.join(*path)

    def link(self, *args):
        return self.path(*args, is_link=True)

    def abs_link(self, dst):
        # Normalize
        dst = urljoin(self.config['BLOG_URL'], dst)

        return urlparse(dst).path

    def rel_link(self, src, dst):
        # Normalize
        src = urljoin(self.config['BLOG_URL'], src)
        dst = urljoin(src, dst)
        # Avoid empty links.
        if src == dst:
            return "#"
        # Check that link can be made relative, otherwise return dest
        parsed_src = urlsplit(src)
        parsed_dst = urlsplit(dst)
        if parsed_src[:2] != parsed_dst[:2]:
            return dst
        # Now both paths are on the same site and absolute
        src_elems = parsed_src.path.split('/')[1:]
        dst_elems = parsed_dst.path.split('/')[1:]
        i = 0
        for (i, s), d in zip(enumerate(src_elems), dst_elems):
            if s != d:
                break
        else:
            i += 1
        # Now i is the longest common prefix
        return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])

    def file_exists(self, path, not_empty=False):
        """Returns True if the file exists. If not_empty is True,
        it also has to be not empty."""
        exists = os.path.exists(path)
        if exists and not_empty:
            exists = os.stat(path).st_size > 0
        return exists

    def gen_tasks(self):

        def create_gzipped_copy(in_path, out_path):
            with gzip.GzipFile(out_path, 'wb+') as outf:
                with open(in_path, 'rb') as inf:
                    outf.write(inf.read())

        def flatten(task):
            if isinstance(task, dict):
                yield task
            else:
                for t in task:
                    for ft in flatten(t):
                        yield ft

        def add_gzipped_copies(task):
            if not self.config['GZIP_FILES']:
                return None
            gzip_task = {
                'file_dep': [],
                'targets': [],
                'actions': [],
                'basename': 'gzip',
                'name': task.get('name', 'unknown'),
                'clean': True,
            }
            targets = task.get('targets', [])
            flag = False
            for target in targets:
                ext = os.path.splitext(target)[1]
                if (ext.lower() in self.config['GZIP_EXTENSIONS'] and
                        target.startswith(self.config['OUTPUT_FOLDER'])):
                    flag = True
                    gzipped = target + '.gz'
                    gzip_task['file_dep'].append(target)
                    gzip_task['targets'].append(gzipped)
                    gzip_task['actions'].append((create_gzipped_copy, (target, gzipped)))
            if not flag:
                return None
            return gzip_task

        if self.config['GZIP_FILES']:
            task_dep = ['gzip']
        else:
            task_dep = []
        for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
            for task in flatten(pluginInfo.plugin_object.gen_tasks()):
                gztask = add_gzipped_copies(task)
                if gztask:
                    yield gztask
                yield task
            if pluginInfo.plugin_object.is_default:
                task_dep.append(pluginInfo.plugin_object.name)

        for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
            for task in pluginInfo.plugin_object.gen_tasks():
                gztask = add_gzipped_copies(task)
                if gztask:
                    yield gztask
                yield task
            if pluginInfo.plugin_object.is_default:
                task_dep.append(pluginInfo.plugin_object.name)
        yield {
            'name': b'all',
            'actions': None,
            'clean': True,
            'task_dep': task_dep
        }

    def scan_posts(self):
        """Scan all the posts."""
        if self._scanned:
            return

        print("Scanning posts", end='')
        tzinfo = None
        if self.config['TIMEZONE'] is not None:
            tzinfo = pytz.timezone(self.config['TIMEZONE'])
        targets = set([])
        for wildcard, destination, template_name, use_in_feeds in \
                self.config['post_pages']:
            print(".", end='')
            dirname = os.path.dirname(wildcard)
            for dirpath, _, _ in os.walk(dirname):
                dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
                dest_dir = os.path.normpath(os.path.join(dirpath,
                                            os.path.relpath(dirpath, dirname)))
                for base_path in glob.glob(dir_glob):
                    post = Post(
                        base_path,
                        self.config['CACHE_FOLDER'],
                        dest_dir,
                        use_in_feeds,
                        self.config['TRANSLATIONS'],
                        self.config['DEFAULT_LANG'],
                        self.config['BLOG_URL'],
                        self.MESSAGES,
                        template_name,
                        self.config['FILE_METADATA_REGEXP'],
                        tzinfo,
                    )
                    for lang, langpath in list(
                            self.config['TRANSLATIONS'].items()):
                        dest = (destination, langpath, dir_glob,
                                post.pagenames[lang])
                        if dest in targets:
                            raise Exception('Duplicated output path {0!r} '
                                            'in post {1!r}'.format(
                                                post.pagenames[lang],
                                                base_path))
                        targets.add(dest)
                    self.global_data[post.post_name] = post
                    if post.use_in_feeds:
                        self.posts_per_year[
                            str(post.date.year)].append(post.post_name)
                        for tag in post.tags:<commit_msg>Typo in destination building<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012 <NAME> otros.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import print_function, unicode_literals
from collections import defaultdict
from copy import copy
import glob
import gzip
import os
import sys
try:
    from urlparse import urlparse, urlsplit, urljoin
except ImportError:
    from urllib.parse import urlparse, urlsplit, urljoin  # NOQA

import lxml.html
from yapsy.PluginManager import PluginManager
import pytz

if os.getenv('DEBUG'):
    import logging
    logging.basicConfig(level=logging.DEBUG)
else:
    import logging
    logging.basicConfig(level=logging.ERROR)

from .post import Post
from . import utils
from .plugin_categories import (
    Command,
    LateTask,
    PageCompiler,
    Task,
    TemplateSystem,
)

config_changed = utils.config_changed

__all__ = ['Nikola']


class Nikola(object):

    """Class that handles site generation.

    Takes a site config as argument on creation.
    """

    def __init__(self, **config):
        """Setup proper environment for running tasks."""

        self.global_data = {}
        self.posts_per_year = defaultdict(list)
        self.posts_per_tag = defaultdict(list)
        self.timeline = []
        self.pages = []
        self._scanned = False
        if not config:
            self.configured = False
        else:
            self.configured = True

        # This is the default config
        # TODO: fill it
        self.config = {
            'ADD_THIS_BUTTONS': True,
            'ANALYTICS': '',
            'ARCHIVE_PATH': "",
            'ARCHIVE_FILENAME': "archive.html",
            'CACHE_FOLDER': 'cache',
            'COMMENTS_IN_GALLERIES': False,
            'COMMENTS_IN_STORIES': False,
            'CONTENT_FOOTER': '',
            'DATE_FORMAT': '%Y-%m-%d %H:%M',
            'DEFAULT_LANG': "en",
            'DEPLOY_COMMANDS': [],
            'DISQUS_FORUM': 'nikolademo',
            'FAVICONS': {},
            'FILE_METADATA_REGEXP': None,
            'FILES_FOLDERS': {'files': ''},
            'FILTERS': {},
            'GALLERY_PATH': 'galleries',
            'GZIP_FILES': False,
            'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js'),
            'INDEX_DISPLAY_POST_COUNT': 10,
            'INDEX_TEASERS': False,
            'INDEXES_TITLE': "",
            'INDEXES_PAGES': "",
            'INDEX_PATH': '',
            'LICENSE': '',
            'LISTINGS_FOLDER': 'listings',
            'MAX_IMAGE_SIZE': 1280,
            'MATHJAX_CONFIG': '',
            'OUTPUT_FOLDER': 'output',
            'post_compilers': {
                "rest": ('.txt', '.rst'),
                "markdown": ('.md', '.mdown', '.markdown'),
                "textile": ('.textile',),
                "txt2tags": ('.t2t',),
                "bbcode": ('.bb',),
                "wiki": ('.wiki',),
                "ipynb": ('.ipynb',),
                "html": ('.html', '.htm')
            },
            'POST_PAGES': (
                ("posts/*.txt", "posts", "post.tmpl", True),
                ("stories/*.txt", "stories", "story.tmpl", False),
            ),
            'REDIRECTIONS': [],
            'RSS_LINK': None,
            'RSS_PATH': '',
            'RSS_TEASERS': True,
            'SEARCH_FORM': '',
            'SLUG_TAG_PATH': True,
            'STORY_INDEX': False,
            'TAG_PATH': 'categories',
            'TAG_PAGES_ARE_INDEXES': False,
            'THEME': 'site',
            'THUMBNAIL_SIZE': 180,
            'USE_BUNDLES': True,
            'USE_CDN': False,
            'USE_FILENAME_AS_TITLE': True,
            'TIMEZONE': None,
        }

        self.config.update(config)
        self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
                                                      {self.config['DEFAULT_'
                                                      'LANG']: ''})

        self.THEMES = utils.get_theme_chain(self.config['THEME'])

        self.MESSAGES = utils.load_messages(self.THEMES,
                                            self.config['TRANSLATIONS'])

        self.plugin_manager = PluginManager(categories_filter={
            "Command": Command,
            "Task": Task,
            "LateTask": LateTask,
            "TemplateSystem": TemplateSystem,
            "PageCompiler": PageCompiler,
        })
        self.plugin_manager.setPluginInfoExtension('plugin')
        self.plugin_manager.setPluginPlaces([
            str(os.path.join(os.path.dirname(__file__), 'plugins')),
            str(os.path.join(os.getcwd(), 'plugins')),
        ])
        self.plugin_manager.collectPlugins()

        self.commands = {}
        # Activate all command plugins
        for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
            self.plugin_manager.activatePluginByName(pluginInfo.name)
            pluginInfo.plugin_object.set_site(self)
            pluginInfo.plugin_object.short_help = pluginInfo.description
            self.commands[pluginInfo.name] = pluginInfo.plugin_object

        # Activate all task plugins
        for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
            self.plugin_manager.activatePluginByName(pluginInfo.name)
            pluginInfo.plugin_object.set_site(self)

        for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
            self.plugin_manager.activatePluginByName(pluginInfo.name)
            pluginInfo.plugin_object.set_site(self)

        # set global_context for template rendering
        self.GLOBAL_CONTEXT = {
        }

        self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
        self.GLOBAL_CONTEXT['_link'] = self.link
        self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
        self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
        self.GLOBAL_CONTEXT['exists'] = self.file_exists
        self.GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config[
            'SLUG_TAG_PATH']

        self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
            'ADD_THIS_BUTTONS']
        self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
            'INDEX_DISPLAY_POST_COUNT']
        self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
        self.GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
        self.GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
        self.GLOBAL_CONTEXT['date_format'] = self.config.get(
            'DATE_FORMAT', '%Y-%m-%d %H:%M')
        self.GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
        self.GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
        self.GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL')
        if (self.GLOBAL_CONTEXT['blog_url'] is None and
                self.config.get('BLOG_URL')):
            # TODO: warn if using old options
            # print("WARNING: You should configure SITE_URL instead of BLOG_URL")
            # print("See docs at FIXME put URL")
            self.GLOBAL_CONTEXT['blog_url'] = self.config.get('BLOG_URL')
        else:  # FIXME: temporary until I get rid of BLOG_URL
            self.config['BLOG_URL'] = self.config.get('SITE_URL')
        self.GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
        self.GLOBAL_CONTEXT['analytics'] = self.config.get('ANALYTICS')
        self.GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
        self.GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
        self.GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
        self.GLOBAL_CONTEXT['disqus_forum'] = self.config.get('DISQUS_FORUM')
        self.GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
            'MATHJAX_CONFIG')
        self.GLOBAL_CONTEXT['content_footer'] = self.config.get(
            'CONTENT_FOOTER')
        self.GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
        self.GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
        self.GLOBAL_CONTEXT['sidebar_links'] = self.config.get('SIDEBAR_LINKS')
        self.GLOBAL_CONTEXT['twitter_card'] = self.config.get(
            'TWITTER_CARD', {})

        self.GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))

        # check if custom css exist and is not empty
        for files_path in list(self.config['FILES_FOLDERS'].keys()):
            custom_css_path = os.path.join(files_path, 'assets/css/custom.css')
            if self.file_exists(custom_css_path, not_empty=True):
                self.GLOBAL_CONTEXT['has_custom_css'] = True
                break
        else:
            self.GLOBAL_CONTEXT['has_custom_css'] = False

        # Load template plugin
        template_sys_name = utils.get_template_engine(self.THEMES)
        pi = self.plugin_manager.getPluginByName(
            template_sys_name, "TemplateSystem")
        if pi is None:
            sys.stderr.write("Error loading {0} template system "
                             "plugin\n".format(template_sys_name))
            sys.exit(1)
        self.template_system = pi.plugin_object
        lookup_dirs = [os.path.join(utils.get_theme_path(name), "templates")
                       for name in self.THEMES]
        self.template_system.set_directories(lookup_dirs,
                                             self.config['CACHE_FOLDER'])

        # Load compiler plugins
        self.compilers = {}
        self.inverse_compilers = {}

        for pluginInfo in self.plugin_manager.getPluginsOfCategory(
                "PageCompiler"):
            self.compilers[pluginInfo.name] = \
                pluginInfo.plugin_object.compile_html

    def get_compiler(self, source_name):
        """Get the correct compiler for a post from `conf.post_compilers`

        To make things easier for users, the mapping in conf.py is
        compiler->[extensions], although this is less convenient for us. The
        majority of this function is reversing that dictionary and error
        checking.
        """
        ext = os.path.splitext(source_name)[1]
        try:
            compile_html = self.inverse_compilers[ext]
        except KeyError:
            # Find the correct compiler for this files extension
            langs = [lang for lang, exts in
                     list(self.config['post_compilers'].items())
                     if ext in exts]
            if len(langs) != 1:
                if len(set(langs)) > 1:
                    exit("Your file extension->compiler definition is"
                         "ambiguous.\nPlease remove one of the file extensions"
                         "from 'post_compilers' in conf.py\n(The error is in"
                         "one of {0})".format(', '.join(langs)))
                elif len(langs) > 1:
                    langs = langs[:1]
                else:
                    exit("post_compilers in conf.py does not tell me how to "
                         "handle '{0}' extensions.".format(ext))

            lang = langs[0]
            compile_html = self.compilers[lang]
            self.inverse_compilers[ext] = compile_html

        return compile_html

    def render_template(self, template_name, output_name, context):
        local_context = {}
        local_context["template_name"] = template_name
        local_context.update(self.GLOBAL_CONTEXT)
        local_context.update(context)
        data = self.template_system.render_template(
            template_name, None, local_context)

        assert isinstance(output_name, bytes)
        assert output_name.startswith(
            self.config["OUTPUT_FOLDER"].encode('utf8'))
        url_part = output_name.decode('utf8')[len(self.config["OUTPUT_FOLDER"])
                                              + 1:]

        # This is to support windows paths
        url_part = "/".join(url_part.split(os.sep))

        src = urljoin(self.config["BLOG_URL"], url_part)

        parsed_src = urlsplit(src)
        src_elems = parsed_src.path.split('/')[1:]

        def replacer(dst):
            # Refuse to replace links that are full URLs.
            dst_url = urlparse(dst)
            if dst_url.netloc:
                if dst_url.scheme == 'link':  # Magic link
                    dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
                                    context['lang'])
                else:
                    return dst

            # Normalize
            dst = urljoin(src, dst)
            # Avoid empty links.
            if src == dst:
                return "#"
            # Check that link can be made relative, otherwise return dest
            parsed_dst = urlsplit(dst)
            if parsed_src[:2] != parsed_dst[:2]:
                return dst

            # Now both paths are on the same site and absolute
            dst_elems = parsed_dst.path.split('/')[1:]

            i = 0
            for (i, s), d in zip(enumerate(src_elems), dst_elems):
                if s != d:
                    break
            # Now i is the longest common prefix
            result = '/'.join(['..'] * (len(src_elems) - i - 1) +
                              dst_elems[i:])

            if not result:
                result = "."

            # Don't forget the fragment (anchor) part of the link
            if parsed_dst.fragment:
                result += "#" + parsed_dst.fragment

            assert result, (src, dst, i, src_elems, dst_elems)

            return result

        try:
            os.makedirs(os.path.dirname(output_name))
        except:
            pass
        doc = lxml.html.document_fromstring(data)
        doc.rewrite_links(replacer)
        data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
        with open(output_name, "wb+") as post_file:
            post_file.write(data)

    def path(self, kind, name, lang, is_link=False):
        """Build the path to a certain kind of page.

        kind is one of:

        * tag_index (name is ignored)
        * tag (and name is the tag name)
        * tag_rss (name is the tag name)
        * archive (and name is the year, or None for the main archive index)
        * index (name is the number in index-number)
        * rss (name is ignored)
        * gallery (name is the gallery name)
        * listing (name is the source code file name)
        * post_path (name is 1st element in a post_pages tuple)

        The returned value is always a path relative to output, like
        "categories/whatever.html"

        If is_link is True, the path is absolute and uses "/" as separator
        (ex: "/archive/index.html").
        If is_link is False, the path is relative to output and uses the
        platform's separator.
        (ex: "archive\\index.html")
        """

        path = []

        if kind == "tag_index":
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['TAG_PATH'], 'index.html'] if _f]
        elif kind == "tag":
            if self.config['SLUG_TAG_PATH']:
                name = utils.slugify(name)
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['TAG_PATH'], name + ".html"] if
                    _f]
        elif kind == "tag_rss":
            if self.config['SLUG_TAG_PATH']:
                name = utils.slugify(name)
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['TAG_PATH'], name + ".xml"] if
                    _f]
        elif kind == "index":
            if name not in [None, 0]:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['INDEX_PATH'],
                                      'index-{0}.html'.format(name)] if _f]
            else:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['INDEX_PATH'], 'index.html']
                        if _f]
        elif kind == "post_path":
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  os.path.dirname(name), "index.html"] if _f]
        elif kind == "rss":
            path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                  self.config['RSS_PATH'], 'rss.xml'] if _f]
        elif kind == "archive":
            if name:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['ARCHIVE_PATH'], name,
                                      'index.html'] if _f]
            else:
                path = [_f for _f in [self.config['TRANSLATIONS'][lang],
                                      self.config['ARCHIVE_PATH'],
                                      self.config['ARCHIVE_FILENAME']] if _f]
        elif kind == "gallery":
            path = [_f for _f in [self.config['GALLERY_PATH'], name,
                                  'index.html'] if _f]
        elif kind == "listing":
            path = [_f for _f in [self.config['LISTINGS_FOLDER'], name +
                                  '.html'] if _f]
        if is_link:
            return '/' + ('/'.join(path))
        else:
            return os.path.join(*path)

    def link(self, *args):
        return self.path(*args, is_link=True)

    def abs_link(self, dst):
        # Normalize
        dst = urljoin(self.config['BLOG_URL'], dst)

        return urlparse(dst).path

    def rel_link(self, src, dst):
        # Normalize
        src = urljoin(self.config['BLOG_URL'], src)
        dst = urljoin(src, dst)
        # Avoid empty links.
        if src == dst:
            return "#"
        # Check that link can be made relative, otherwise return dest
        parsed_src = urlsplit(src)
        parsed_dst = urlsplit(dst)
        if parsed_src[:2] != parsed_dst[:2]:
            return dst
        # Now both paths are on the same site and absolute
        src_elems = parsed_src.path.split('/')[1:]
        dst_elems = parsed_dst.path.split('/')[1:]
        i = 0
        for (i, s), d in zip(enumerate(src_elems), dst_elems):
            if s != d:
                break
        else:
            i += 1
        # Now i is the longest common prefix
        return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])

    def file_exists(self, path, not_empty=False):
        """Returns True if the file exists. If not_empty is True,
        it also has to be not empty."""
        exists = os.path.exists(path)
        if exists and not_empty:
            exists = os.stat(path).st_size > 0
        return exists

    def gen_tasks(self):

        def create_gzipped_copy(in_path, out_path):
            with gzip.GzipFile(out_path, 'wb+') as outf:
                with open(in_path, 'rb') as inf:
                    outf.write(inf.read())

        def flatten(task):
            if isinstance(task, dict):
                yield task
            else:
                for t in task:
                    for ft in flatten(t):
                        yield ft

        def add_gzipped_copies(task):
            if not self.config['GZIP_FILES']:
                return None
            gzip_task = {
                'file_dep': [],
                'targets': [],
                'actions': [],
                'basename': 'gzip',
                'name': task.get('name', 'unknown'),
                'clean': True,
            }
            targets = task.get('targets', [])
            flag = False
            for target in targets:
                ext = os.path.splitext(target)[1]
                if (ext.lower() in self.config['GZIP_EXTENSIONS'] and
                        target.startswith(self.config['OUTPUT_FOLDER'])):
                    flag = True
                    gzipped = target + '.gz'
                    gzip_task['file_dep'].append(target)
                    gzip_task['targets'].append(gzipped)
                    gzip_task['actions'].append((create_gzipped_copy, (target, gzipped)))
            if not flag:
                return None
            return gzip_task

        if self.config['GZIP_FILES']:
            task_dep = ['gzip']
        else:
            task_dep = []
        for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
            for task in flatten(pluginInfo.plugin_object.gen_tasks()):
                gztask = add_gzipped_copies(task)
                if gztask:
                    yield gztask
                yield task
            if pluginInfo.plugin_object.is_default:
                task_dep.append(pluginInfo.plugin_object.name)

        for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
            for task in pluginInfo.plugin_object.gen_tasks():
                gztask = add_gzipped_copies(task)
                if gztask:
                    yield gztask
                yield task
            if pluginInfo.plugin_object.is_default:
                task_dep.append(pluginInfo.plugin_object.name)
        yield {
            'name': b'all',
            'actions': None,
            'clean': True,
            'task_dep': task_dep
        }

    def scan_posts(self):
        """Scan all the posts."""
        if self._scanned:
            return

        print("Scanning posts", end='')
        tzinfo = None
        if self.config['TIMEZONE'] is not None:
            tzinfo = pytz.timezone(self.config['TIMEZONE'])
        targets = set([])
        for wildcard, destination, template_name, use_in_feeds in \
                self.config['post_pages']:
            print(".", end='')
            dirname = os.path.dirname(wildcard)
            for dirpath, _, _ in os.walk(dirname):
                dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
                dest_dir = os.path.normpath(os.path.join(destination,
                                            os.path.relpath(dirpath, dirname)))
                for base_path in glob.glob(dir_glob):
                    post = Post(
                        base_path,
                        self.config['CACHE_FOLDER'],
                        dest_dir,
                        use_in_feeds,
                        self.config['TRANSLATIONS'],
                        self.config['DEFAULT_LANG'],
                        self.config['BLOG_URL'],
                        self.MESSAGES,
                        template_name,
                        self.config['FILE_METADATA_REGEXP'],
                        tzinfo,
                    )
                    for lang, langpath in list(
                            self.config['TRANSLATIONS'].items()):
                        dest = (destination, langpath, dir_glob,
                                post.pagenames[lang])
                        if dest in targets:
                            raise Exception('Duplicated output path {0!r} '
                                            'in post {1!r}'.format(
                                                post.pagenames[lang],
                                                base_path))
                        targets.add(dest)
                    self.global_data[post.post_name] = post
                    if post.use_in_feeds:
                        self.posts_per_year[
                            str(post.date.year)].append(post.post_name)
                        for tag in post.tags:
{'fraction_non_alphanumeric': 0.11008873496321495, 'fraction_numerical': 0.0020493063098141278, 'mean_word_length': 1.7048389778837094, 'pattern_counts': {'":': 44, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 66, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6463488', 'n_tokens_mistral': 13869, 'n_tokens_neox': 13442, 'n_words': 3189}