<commit_before># Copyright 2014 <NAME>
#
# This file is part of Urubu.
#
# Urubu is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Urubu is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Urubu.  If not, see <http://www.gnu.org/licenses/>.

# Python 3 idioms
from __future__ import unicode_literals
from io import open
import os, sys, json, itertools

import markdown
import logging
logging.captureWarnings(False)
from bs4 import BeautifulSoup

import jinja2

from urubu import UrubuWarning, UrubuError, urubu_warn, _warning
from urubu import md_extensions

from urubu.config import layoutdir, tag_layout, tipuesearchdir, tipuesearch_content
from urubu._compat import text_type

def skip_yamlfm(f):
    """Return source of a file without yaml frontmatter."""
    f.readline()
    found = False
    lines = []
    for line in f.readlines():
        if found:
            lines.append(line)
        if line.strip() == '---':
            found = True
    return ''.join(lines)


class ContentProcessor(object):

    def __init__(self, sitedir, project):
        self.sitedir = sitedir
        self.filelist = project.filelist
        self.navlist = project.navlist
        self.taglist = project.taglist
        self.site = project.site
        dlclass = md_extensions.DLClassExtension()
        tableclass = md_extensions.TableClassExtension()
        projectref = md_extensions.ProjectReferenceExtension()
        extractanchors = md_extensions.ExtractAnchorsExtension()
        marktag = md_extensions.MarkTagExtension()
        # there is a strange interaction between smarty and reference links that start on a new line
        # disabling smarty for now...
        # extensions = ['extra', 'codehilite', 'headerid', 'toc', 'smarty', tableclass, projectref]
        extensions = ['extra', 'codehilite', 'toc',
                      dlclass, tableclass, projectref, extractanchors]
        if self.site['mark_tag_support']:
            extensions.append(marktag)
        extension_configs = {'codehilite': [('guess_lang', 'False'),
                                            ('linenums', 'False')],
                             'toc': [('baselevel', 2)]
                             }
        self.md = markdown.Markdown(extensions=extensions,
                                    extension_configs=extension_configs)
        self.md.site = self.site
        self.md.anchors = project.anchors
        if 'strict_undefined' in self.site and self.site['strict_undefined']:
            env = self.env = jinja2.Environment(
                loader=jinja2.FileSystemLoader(layoutdir),
                lstrip_blocks=True,
                trim_blocks=True,
                undefined=jinja2.StrictUndefined)
        else:
            env = self.env = jinja2.Environment(
                loader=jinja2.FileSystemLoader(layoutdir),
                lstrip_blocks=True,
                trim_blocks=True)
        env.filters.update(project.filters)
        self.templates = {}
        for layout in project.layouts:
            self.templates[layout] = self.env.get_template(layout + '.html')
        # layout for tags is optional, triggers index file generation per tag
        try:
            self.templates[tag_layout] = self.env.get_template(
                tag_layout + '.html')
        except jinja2.exceptions.TemplateNotFound:
            if self.taglist:
                urubu_warn(_warning.undef_tag_layout, msg=tag_layout)

    def process(self):
        """Process the content.

        Conversion and rendering are done in separate phases, so
        that the full content is available to the rendering process.
        """
        self.convert()
        self.render()
        self.make_tipuesearch_content()

    def convert(self):
        for info in self.filelist:
            fn = info['fn']
            with open(fn, encoding='utf-8-sig') as inf:
                src = skip_yamlfm(inf)
            self.md.this = info
            # first process as a template<commit_msg>Remove duplicated code setting up the environment<commit_after># Copyright 2014 <NAME>
#
# This file is part of Urubu.
#
# Urubu is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Urubu is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Urubu.  If not, see <http://www.gnu.org/licenses/>.

# Python 3 idioms
from __future__ import unicode_literals
from io import open
import os, sys, json, itertools

import markdown
import logging
logging.captureWarnings(False)
from bs4 import BeautifulSoup

import jinja2

from urubu import UrubuWarning, UrubuError, urubu_warn, _warning
from urubu import md_extensions

from urubu.config import layoutdir, tag_layout, tipuesearchdir, tipuesearch_content
from urubu._compat import text_type

def skip_yamlfm(f):
    """Return source of a file without yaml frontmatter."""
    f.readline()
    found = False
    lines = []
    for line in f.readlines():
        if found:
            lines.append(line)
        if line.strip() == '---':
            found = True
    return ''.join(lines)


class ContentProcessor(object):

    def __init__(self, sitedir, project):
        self.sitedir = sitedir
        self.filelist = project.filelist
        self.navlist = project.navlist
        self.taglist = project.taglist
        self.site = project.site
        dlclass = md_extensions.DLClassExtension()
        tableclass = md_extensions.TableClassExtension()
        projectref = md_extensions.ProjectReferenceExtension()
        extractanchors = md_extensions.ExtractAnchorsExtension()
        marktag = md_extensions.MarkTagExtension()
        # there is a strange interaction between smarty and reference links that start on a new line
        # disabling smarty for now...
        # extensions = ['extra', 'codehilite', 'headerid', 'toc', 'smarty', tableclass, projectref]
        extensions = ['extra', 'codehilite', 'toc',
                      dlclass, tableclass, projectref, extractanchors]
        if self.site['mark_tag_support']:
            extensions.append(marktag)
        extension_configs = {'codehilite': [('guess_lang', 'False'),
                                            ('linenums', 'False')],
                             'toc': [('baselevel', 2)]
                             }
        self.md = markdown.Markdown(extensions=extensions,
                                    extension_configs=extension_configs)
        self.md.site = self.site
        self.md.anchors = project.anchors
        if 'strict_undefined' in self.site and self.site['strict_undefined']:
            undefined_class = jinja2.StrictUndefined
        else:
            undefined_class = jinja2.Undefined
        env = self.env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(layoutdir),
            lstrip_blocks=True,
            trim_blocks=True,
            undefined=undefined_class)
        env.filters.update(project.filters)
        self.templates = {}
        for layout in project.layouts:
            self.templates[layout] = self.env.get_template(layout + '.html')
        # layout for tags is optional, triggers index file generation per tag
        try:
            self.templates[tag_layout] = self.env.get_template(
                tag_layout + '.html')
        except jinja2.exceptions.TemplateNotFound:
            if self.taglist:
                urubu_warn(_warning.undef_tag_layout, msg=tag_layout)

    def process(self):
        """Process the content.

        Conversion and rendering are done in separate phases, so
        that the full content is available to the rendering process.
        """
        self.convert()
        self.render()
        self.make_tipuesearch_content()

    def convert(self):
        for info in self.filelist:
            fn = info['fn']
            with open(fn, encoding='utf-8-sig') as inf:
                src = skip_yamlfm(inf)
            self.md.this = info
            # first process as a template
{'fraction_non_alphanumeric': 0.08468427615916563, 'fraction_numerical': 0.0035143407776896045, 'mean_word_length': 2.4074932406334493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781991', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2343, 'n_words': 735}