<commit_before># -*- coding: utf-8 -*-
"""post and postlist directives."""

import os
import sys
from string import Formatter
from datetime import datetime

from docutils import nodes
from sphinx.locale import _
from sphinx.util.nodes import set_source_info
from sphinx.util.compat import Directive, make_admonition
from docutils.parsers.rst import directives

import ablog
from .blog import Blog, slugify

if sys.version_info >= (3, 0):
    text_type = str
else:
    text_type = unicode


class PostNode(nodes.Element):
    """Represent ``post`` directive content and options in document tree."""

    pass


class PostList(nodes.General, nodes.Element):
    """Represent ``postlist`` directive converted to a list of links."""

    pass


class UpdateNode(nodes.Admonition, nodes.Element):
    """Represent ``update`` directive."""

    pass


class PostDirective(Directive):
    """Handle ``post`` directives."""

    _split = lambda a: [s.strip() for s in (a or '').split(',') if s.strip()]

    has_content = True
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = True
    option_spec = {
        'tags': _split,
        'author': _split,
        'category': _split,
        'location': _split,
        'language': _split,
        'redirect': _split,
        'title': lambda a: a.strip(),
        'image': int,
        'excerpt': int,
        'exclude': directives.flag,
    }

    def run(self):

        node = PostNode()
        node.document = self.state.document
        set_source_info(self, node)
        self.state.nested_parse(self.content, self.content_offset,
                                node, match_titles=1)

        node['date'] = self.arguments[0] if self.arguments else None
        node['tags'] = self.options.get('tags', [])
        node['author'] = self.options.get('author', [])
        node['category'] = self.options.get('category', [])
        node['location'] = self.options.get('location', [])
        node['language'] = self.options.get('language', [])
        node['redirect'] = self.options.get('redirect', [])
        node['title'] = self.options.get('title', None)
        node['image'] = self.options.get('image', None)
        node['excerpt'] = self.options.get('excerpt', None)
        node['exclude'] = 'exclude' in self.options
        return [node]


class UpdateDirective(Directive):

    has_content = True
    required_arguments = 1
    optional_arguments = 0#1
    final_argument_whitespace = True
    option_spec = {}

    def run(self):

        ad = make_admonition(UpdateNode, self.name, [_('Updated on')],
                             self.options,
                             self.content, self.lineno, self.content_offset,
                             self.block_text, self.state, self.state_machine)
        ad[0]['date'] = self.arguments[0] if self.arguments else ''
        set_source_info(self, ad[0])
        return ad


class PostListDirective(Directive):
    """Handle ``postlist`` directives."""

    _split = lambda a: set(s.strip() for s in a.split(','))
    has_content = False
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = False
    option_spec = {
        'tags': _split,
        'author': _split,
        'category': _split,
        'location': _split,
        'language': _split,
        'format': lambda a: a.strip(),
        'date': lambda a: a.strip(),
        'sort': directives.flag,
        'excerpts': directives.flag,
        'list-style': lambda a: a.strip(),
    }

    def run(self):

        node = PostList()
        node.document = self.state.document
        set_source_info(self, node)
        self.state.nested_parse(self.content, self.content_offset,
                                node, match_titles=1)

        node['length'] = int(self.arguments[0]) if self.arguments else None
        node['tags'] = self.options.get('tags', [])
        node['author'] = self.options.get('author', [])
        node['category'] = self.options.get('category', [])
        node['location'] = self.options.get('location', [])
        node['language'] = self.options.get('language', [])
        node['format'] = self.options.get('format', '{date} - {title}')
        node['date'] = self.options.get('date', None)
        node['sort'] = 'sort' in self.options
        node['excerpts'] = 'excerpts' in self.options
        node['image'] = 'image' in self.options
        node['list-style'] = self.options.get('list-style', 'none')
        return [node]


def purge_posts(app, env, docname):
    """Remove post and reference to it from the standard domain when its
    document is removed or changed."""

    if hasattr(env, 'ablog_posts'):
        env.ablog_posts.pop(docname, None)

    filename = os.path.split(docname)[1]
    env.domains['std'].data['labels'].pop(filename, None)


def _get_section_title(section):
    """Return section title as text."""

    for title in section.traverse(nodes.title):
        break
    # A problem with the following is that title may contain pending
    # references, e.g. :ref:`tag-tips`
    return title.astext()


def _get_update_dates(section, docname, post_date_format):
    """Return list of dates of updates found section."""


    update_nodes = list(section.traverse(UpdateNode))
    update_dates = []
    for update_node in update_nodes:
        try:
            update = datetime.strptime(update_node['date'], post_date_format)
        except ValueError:
            raise ValueError('invalid post update date in: ' + docname)

        substitute = nodes.title(u'',
            update_node[0][0].astext() + u' ' + update.strftime(post_date_format))
        update_node[0].replace_self(substitute)
        # for now, let updates look like note
        update_node['classes'] = ['note', 'update']

        update_dates.append(update)
    return update_dates


def process_posts(app, doctree):
    """Process posts and map posted document names to post details in the
    environment."""

    env = app.builder.env
    if os.environ.get('READTHEDOCS', None) == 'True':
        env.topickle = lambda *args: env.warn('index',
            'Building on Read The Docs, environment is not being pickled.')
    if not hasattr(env, 'ablog_posts'):
        env.ablog_posts = {}

    post_nodes = list(doctree.traverse(PostNode))
    if not post_nodes:
        return
    post_date_format = app.config['post_date_format']
    docname = env.docname

    # mark the post as 'orphan' so that
    #   "document isn't included in any toctree" warning is not issued
    app.env.metadata[docname]['orphan'] = True

    blog = Blog(app)
    auto_excerpt = blog.post_auto_excerpt
    multi_post = len(post_nodes) > 1 or blog.post_always_section<commit_msg>Introspecting env on rtd.<commit_after># -*- coding: utf-8 -*-
"""post and postlist directives."""

import os
import sys
from string import Formatter
from datetime import datetime

from docutils import nodes
from sphinx.locale import _
from sphinx.util.nodes import set_source_info
from sphinx.util.compat import Directive, make_admonition
from docutils.parsers.rst import directives

import ablog
from .blog import Blog, slugify

if sys.version_info >= (3, 0):
    text_type = str
else:
    text_type = unicode


class PostNode(nodes.Element):
    """Represent ``post`` directive content and options in document tree."""

    pass


class PostList(nodes.General, nodes.Element):
    """Represent ``postlist`` directive converted to a list of links."""

    pass


class UpdateNode(nodes.Admonition, nodes.Element):
    """Represent ``update`` directive."""

    pass


class PostDirective(Directive):
    """Handle ``post`` directives."""

    _split = lambda a: [s.strip() for s in (a or '').split(',') if s.strip()]

    has_content = True
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = True
    option_spec = {
        'tags': _split,
        'author': _split,
        'category': _split,
        'location': _split,
        'language': _split,
        'redirect': _split,
        'title': lambda a: a.strip(),
        'image': int,
        'excerpt': int,
        'exclude': directives.flag,
    }

    def run(self):

        node = PostNode()
        node.document = self.state.document
        set_source_info(self, node)
        self.state.nested_parse(self.content, self.content_offset,
                                node, match_titles=1)

        node['date'] = self.arguments[0] if self.arguments else None
        node['tags'] = self.options.get('tags', [])
        node['author'] = self.options.get('author', [])
        node['category'] = self.options.get('category', [])
        node['location'] = self.options.get('location', [])
        node['language'] = self.options.get('language', [])
        node['redirect'] = self.options.get('redirect', [])
        node['title'] = self.options.get('title', None)
        node['image'] = self.options.get('image', None)
        node['excerpt'] = self.options.get('excerpt', None)
        node['exclude'] = 'exclude' in self.options
        return [node]


class UpdateDirective(Directive):

    has_content = True
    required_arguments = 1
    optional_arguments = 0#1
    final_argument_whitespace = True
    option_spec = {}

    def run(self):

        ad = make_admonition(UpdateNode, self.name, [_('Updated on')],
                             self.options,
                             self.content, self.lineno, self.content_offset,
                             self.block_text, self.state, self.state_machine)
        ad[0]['date'] = self.arguments[0] if self.arguments else ''
        set_source_info(self, ad[0])
        return ad


class PostListDirective(Directive):
    """Handle ``postlist`` directives."""

    _split = lambda a: set(s.strip() for s in a.split(','))
    has_content = False
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = False
    option_spec = {
        'tags': _split,
        'author': _split,
        'category': _split,
        'location': _split,
        'language': _split,
        'format': lambda a: a.strip(),
        'date': lambda a: a.strip(),
        'sort': directives.flag,
        'excerpts': directives.flag,
        'list-style': lambda a: a.strip(),
    }

    def run(self):

        node = PostList()
        node.document = self.state.document
        set_source_info(self, node)
        self.state.nested_parse(self.content, self.content_offset,
                                node, match_titles=1)

        node['length'] = int(self.arguments[0]) if self.arguments else None
        node['tags'] = self.options.get('tags', [])
        node['author'] = self.options.get('author', [])
        node['category'] = self.options.get('category', [])
        node['location'] = self.options.get('location', [])
        node['language'] = self.options.get('language', [])
        node['format'] = self.options.get('format', '{date} - {title}')
        node['date'] = self.options.get('date', None)
        node['sort'] = 'sort' in self.options
        node['excerpts'] = 'excerpts' in self.options
        node['image'] = 'image' in self.options
        node['list-style'] = self.options.get('list-style', 'none')
        return [node]


def purge_posts(app, env, docname):
    """Remove post and reference to it from the standard domain when its
    document is removed or changed."""

    if hasattr(env, 'ablog_posts'):
        env.ablog_posts.pop(docname, None)

    filename = os.path.split(docname)[1]
    env.domains['std'].data['labels'].pop(filename, None)


def _get_section_title(section):
    """Return section title as text."""

    for title in section.traverse(nodes.title):
        break
    # A problem with the following is that title may contain pending
    # references, e.g. :ref:`tag-tips`
    return title.astext()


def _get_update_dates(section, docname, post_date_format):
    """Return list of dates of updates found section."""


    update_nodes = list(section.traverse(UpdateNode))
    update_dates = []
    for update_node in update_nodes:
        try:
            update = datetime.strptime(update_node['date'], post_date_format)
        except ValueError:
            raise ValueError('invalid post update date in: ' + docname)

        substitute = nodes.title(u'',
            update_node[0][0].astext() + u' ' + update.strftime(post_date_format))
        update_node[0].replace_self(substitute)
        # for now, let updates look like note
        update_node['classes'] = ['note', 'update']

        update_dates.append(update)
    return update_dates

def skip_pickling(env):


    def dump(obj, level=0):
        for attr in dir(obj):
            val = getattr(obj, a)
            if isinstance(val, (int, float, str, unicode, list, dict, set)):
                print('{}{}'.format(level * ' ', repr(val)))
            else:
                dump(val, level=level+1)

    env._topickle = env.topickle
    def topickle(self, *args):

        try:
            self._topickle(*args)
        except:
            pass

        dump(env)

    #env.topickle = lambda *args: env.warn('index',
    #    'Building on Read The Docs, environment is not being pickled.')


def process_posts(app, doctree):
    """Process posts and map posted document names to post details in the
    environment."""

    env = app.builder.env
    if os.environ.get('READTHEDOCS', None) == 'True':
        skip_pickling(env)
    if not hasattr(env, 'ablog_posts'):
        env.ablog_posts = {}

    post_nodes = list(doctree.traverse(PostNode))
    if not post_nodes:
        return
    post_date_format = app.config['post_date_format']
    docname = env.docname

    # mark the post as 'orphan' so that
    #   "document isn't included in any toctree" warning is not issued
    app.env.metadata[docname]['orphan'] = True

    blog = Blog(app)
    auto_excerpt = blog.post_auto_excerpt
    multi_post = len(post_nodes) > 1 or blog.post_always_section
{'fraction_non_alphanumeric': 0.13455898132913657, 'fraction_numerical': 0.0032906502611059446, 'mean_word_length': 2.688654353562005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7168042', 'n_tokens_mistral': 4272, 'n_tokens_neox': 4111, 'n_words': 1043}