<commit_before># -*- coding: utf-8 -*-
import sys

from bs4 import BeautifulSoup
from json import dumps
from lektor.build_programs import BuildProgram
from lektor.context import get_ctx
from lektor.db import F
from lektor.pluginsystem import Plugin
from lektor.sourceobj import VirtualSourceObject
from lektor.utils import build_url


PY2 = sys.version_info[0] == 2

if PY2:
    text_type = unicode
else:
    text_type = str


def split_to_list(value):
    return [item.strip() for item in value.split(',') if item.strip()]


def json_url(alt):
    return '/static/js/tipuesearch_content_{}.json'.format(alt)


class TipueSource(VirtualSourceObject):

    def __init__(self, parent, plugin):
        VirtualSourceObject.__init__(self, parent)
        self.plugin = plugin

    @property
    def path(self):
        return build_url([self.parent.path, '@tipue', 'tipuesearch'])

    def __getattr__(self, item):
        try:
            return self.plugin.config_value('tipuesearch', item)
        except KeyError:
            raise AttributeError(item)


class TipueBuilderProgram(BuildProgram):

    def produce_artifacts(self):
        sources = []
        for page in self.get_all_children():
            sources.extend(page.iter_source_filenames())
        self.declare_artifact(
            json_url(self.source.alt),
            sources=sources
        )

    def get_children(self, page):
        results = []
        for model in split_to_list(self.source.models):
            children = page.children
            children.include_undiscoverable(True)
            children = children.filter(F._model == model)
            results.extend(children.all())
        return list(set(results))

    def get_all_children(self):
        pages = [self.source.parent]
        before = 0
        while before != len(pages):
            before = len(pages)
            pages_ = []
            for page in pages:
                pages_.append(page)
                pages_.extend(self.get_children(page))
            pages = set(pages_)

        return pages

    def join_fields(self, page, fields):
        result = text_type()

        for field in split_to_list(fields):
            try:
                value = page[field]
                if type(value) == text_type:
                    result += ' ' + value
                elif hasattr(value, 'html'):
                    soup = BeautifulSoup(value.html, "html.parser")
                    result += ' ' + soup.text
                elif hasattr(value, 'source'):
                    result += ' ' + text_type(value.source)
            except KeyError:
                pass

        return ' '.join(result.split())

    def build_artifact(self, artifact):

        data = {'pages': []}
        for page in self.get_all_children():
            data['pages'].append({
                'title': self.join_fields(page, self.source.title),
                'text': self.join_fields(page, self.source.text),
                'tags': self.join_fields(page, self.source.tags),
                'url': page.url_path
            })

        get_ctx().record_dependency(artifact.artifact_name)

        with artifact.open('wb') as f:
            f.write(dumps(data).encode('utf-8'))


class TipuePlugin(Plugin):<commit_msg>Index the markdown source rather than the rendered output<commit_after># -*- coding: utf-8 -*-
import sys

from bs4 import BeautifulSoup
from json import dumps
from lektor.build_programs import BuildProgram
from lektor.context import get_ctx
from lektor.db import F
from lektor.pluginsystem import Plugin
from lektor.sourceobj import VirtualSourceObject
from lektor.utils import build_url


PY2 = sys.version_info[0] == 2

if PY2:
    text_type = unicode
else:
    text_type = str


def split_to_list(value):
    return [item.strip() for item in value.split(',') if item.strip()]


def json_url(alt):
    return '/static/js/tipuesearch_content_{}.json'.format(alt)


class TipueSource(VirtualSourceObject):

    def __init__(self, parent, plugin):
        VirtualSourceObject.__init__(self, parent)
        self.plugin = plugin

    @property
    def path(self):
        return build_url([self.parent.path, '@tipue', 'tipuesearch'])

    def __getattr__(self, item):
        try:
            return self.plugin.config_value('tipuesearch', item)
        except KeyError:
            raise AttributeError(item)


class TipueBuilderProgram(BuildProgram):

    def produce_artifacts(self):
        sources = []
        for page in self.get_all_children():
            sources.extend(page.iter_source_filenames())
        self.declare_artifact(
            json_url(self.source.alt),
            sources=sources
        )

    def get_children(self, page):
        results = []
        for model in split_to_list(self.source.models):
            children = page.children
            children.include_undiscoverable(True)
            children = children.filter(F._model == model)
            results.extend(children.all())
        return list(set(results))

    def get_all_children(self):
        pages = [self.source.parent]
        before = 0
        while before != len(pages):
            before = len(pages)
            pages_ = []
            for page in pages:
                pages_.append(page)
                pages_.extend(self.get_children(page))
            pages = set(pages_)

        return pages

    def join_fields(self, page, fields):
        result = text_type()

        for field in split_to_list(fields):
            try:
                value = page[field]
                if type(value) == text_type:
                    result += ' ' + value
                elif hasattr(value, 'source'):
                    result += ' ' + text_type(value.source)
                elif hasattr(value, 'html'):
                    soup = BeautifulSoup(value.html, "html.parser")
                    result += ' ' + soup.text
            except KeyError:
                pass

        return ' '.join(result.split())

    def build_artifact(self, artifact):

        data = {'pages': []}
        for page in self.get_all_children():
            data['pages'].append({
                'title': self.join_fields(page, self.source.title),
                'text': self.join_fields(page, self.source.text),
                'tags': self.join_fields(page, self.source.tags),
                'url': page.url_path
            })

        get_ctx().record_dependency(artifact.artifact_name)

        with artifact.open('wb') as f:
            f.write(dumps(data).encode('utf-8'))


class TipuePlugin(Plugin):
{'fraction_non_alphanumeric': 0.1076219512195122, 'fraction_numerical': 0.0024390243902439024, 'mean_word_length': 2.105063890203502, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6588455', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1809, 'n_words': 419}