<commit_before>"""Router for grow documents."""

import os
from grow.performance import docs_loader
from grow.rendering import render_controller
from grow.routing import path_filter as grow_path_filter
from grow.routing import routes as grow_routes


class Error(Exception):
    """Base router error."""
    pass


class MissingStaticConfigError(Exception):
    """Error for missing a configuration for a static file."""
    pass


class Router(object):
    """Router for pods."""

    def __init__(self, pod):
        self.pod = pod
        self._routes = grow_routes.Routes()

    def _preload_and_expand(self, docs, expand=True):
        # Force preload the docs.
        docs_loader.DocsLoader.load(docs)
        docs_loader.DocsLoader.fix_default_locale(self.pod, docs)
        if expand:
            # Will need all of the docs, so expand them out and preload.
            docs = docs_loader.DocsLoader.expand_locales(self.pod, docs)
            docs_loader.DocsLoader.load(docs)
        return docs

    @property
    def routes(self):
        """Routes reflective of the docs."""
        return self._routes

    def add_all(self, concrete=True):
        """Add all documents and static content."""

        self.add_all_docs(concrete=concrete)
        self.add_all_static(concrete=concrete)
        self.add_all_other()

    def add_all_docs(self, concrete=True):
        """Add all pod docs to the router."""
        with self.pod.profile.timer('Router.add_all_docs'):
            docs = []
            doc_basenames = set()
            for collection in self.pod.list_collections():
                for doc in collection.list_docs_unread():
                    if doc.basename in doc_basenames:
                        continue
                    is_default_locale = doc._locale_kwarg == self.pod.podspec.default_locale
                    # Ignore localized names in the files since they will be
                    # picked up when the locales are expanded.
                    if doc.root_pod_path == doc.pod_path or is_default_locale:
                        docs.append(doc)
                        doc_basenames.add(doc.basename)
                    else:
                        # If this document does not exist with the default
                        # locale it still needs to be added.
                        locale_doc = self.pod.get_doc(doc.pod_path, doc.default_locale)
                        if not locale_doc.exists:
                            docs.append(doc)
                            doc_basenames.add(doc.basename)
            docs = self._preload_and_expand(docs, expand=concrete)
            self.add_docs(docs, concrete=concrete)

    def add_all_other(self):
        """Add all pod docs to the router."""
        with self.pod.profile.timer('Router.add_all_other'):
            podspec = self.pod.podspec.get_config()
            if 'error_routes' in podspec:
                for key, error_route in podspec['error_routes'].iteritems():
                    if key == 'default':
                        key = 404

                    self.routes.add('/{}.html'.format(key), RouteInfo('error', {
                        'key': key,
                        'view': error_route,<commit_msg>Adjusting the de-duplication to not affect the concrete routing.<commit_after>"""Router for grow documents."""

import os
from grow.performance import docs_loader
from grow.rendering import render_controller
from grow.routing import path_filter as grow_path_filter
from grow.routing import routes as grow_routes


class Error(Exception):
    """Base router error."""
    pass


class MissingStaticConfigError(Exception):
    """Error for missing a configuration for a static file."""
    pass


class Router(object):
    """Router for pods."""

    def __init__(self, pod):
        self.pod = pod
        self._routes = grow_routes.Routes()

    def _preload_and_expand(self, docs, expand=True):
        # Force preload the docs.
        docs_loader.DocsLoader.load(docs)
        docs_loader.DocsLoader.fix_default_locale(self.pod, docs)
        if expand:
            # Will need all of the docs, so expand them out and preload.
            docs = docs_loader.DocsLoader.expand_locales(self.pod, docs)
            docs_loader.DocsLoader.load(docs)
        return docs

    @property
    def routes(self):
        """Routes reflective of the docs."""
        return self._routes

    def add_all(self, concrete=True):
        """Add all documents and static content."""

        self.add_all_docs(concrete=concrete)
        self.add_all_static(concrete=concrete)
        self.add_all_other()

    def add_all_docs(self, concrete=True):
        """Add all pod docs to the router."""
        with self.pod.profile.timer('Router.add_all_docs'):
            docs = []
            doc_basenames = set()
            for collection in self.pod.list_collections():
                for doc in collection.list_docs_unread():
                    # Skip duplicate documents when using non-concrete routing.
                    if not concrete and doc.basename in doc_basenames:
                        continue
                    is_default_locale = doc._locale_kwarg == self.pod.podspec.default_locale
                    # Ignore localized names in the files since they will be
                    # picked up when the locales are expanded.
                    if doc.root_pod_path == doc.pod_path or is_default_locale:
                        docs.append(doc)
                        doc_basenames.add(doc.basename)
                    else:
                        # If this document does not exist with the default
                        # locale it still needs to be added.
                        locale_doc = self.pod.get_doc(doc.pod_path, doc.default_locale)
                        if not locale_doc.exists:
                            docs.append(doc)
                            doc_basenames.add(doc.basename)
            docs = self._preload_and_expand(docs, expand=concrete)
            self.add_docs(docs, concrete=concrete)

    def add_all_other(self):
        """Add all pod docs to the router."""
        with self.pod.profile.timer('Router.add_all_other'):
            podspec = self.pod.podspec.get_config()
            if 'error_routes' in podspec:
                for key, error_route in podspec['error_routes'].iteritems():
                    if key == 'default':
                        key = 404

                    self.routes.add('/{}.html'.format(key), RouteInfo('error', {
                        'key': key,
                        'view': error_route,
{'fraction_non_alphanumeric': 0.0867655966081163, 'fraction_numerical': 0.0009085402786190188, 'mean_word_length': 1.8742384682332462, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4588452', 'n_tokens_mistral': 1819, 'n_tokens_neox': 1717, 'n_words': 471}