<commit_before>from docset.actions import cached_html, update_toc
from docset.scrape import scrape
import index_rules
import logging
import os
from rust_doc import guide_title
import toc_rules
from types import to_dash_type

log = logging.getLogger('docset.rust')


# Creates a full qualified name based on prefix and name
def make_fqn(prefix, name):
    return "%s::%s" % (prefix, name)


# Appends #ref to path if not None
def path_with_ref(path, ref):
    if ref:
        return path + "#" + ref
    else:
        return path


@cached_html
def add_guide(ctx, tree):
    # This one should work only for nigthly builds
    #
    # Special case, main index "index.html" and "rustdoc.html"
    # have the same title "Rust Documentation", which is rather
    # confusing.
    #
    # What happens here - index is just given title "Index"
    if ctx['rel_path'] == 'index.html':
        ctx['idx'].add("Index", "gd", ctx['rel_path'], to_dash_type)
    else:
        titles = scrape(tree, index_rules.guide_titles())
        if len(titles) > 0:
            ctx['idx'].add(guide_title(titles[0][0]),
                           "gd", ctx['rel_path'], to_dash_type)


@cached_html
def add_module(ctx, tree):
    fqn_prefix = os.path.dirname(ctx['rel_path']).replace(os.sep, "::")
    ctx['idx'].add(fqn_prefix, "mod", ctx['rel_path'], to_dash_type)
    update_toc(ctx, tree, toc_rules.by_type("mod"),
               lambda x: to_dash_type(x, x))


@cached_html
def add_decl_html(ctx, tree):
    idx = ctx['idx']

    rel_path = ctx['rel_path']
    name, _ = os.path.splitext(os.path.basename(rel_path))
    dirname = os.path.dirname(rel_path)

    fqn_prefix = dirname.replace(os.sep, "::")
    name_parts = name.split('.')
    if len(name_parts) < 2:
        log.error("Unexpected HTML file: %s" % rel_path)
        return
    ty, name = name_parts
    fqn = make_fqn(fqn_prefix, name)
<commit_msg>Renaming Index to Overview, minor cleanup<commit_after>from docset.actions import cached_html, update_toc
from docset.scrape import scrape
import index_rules
import logging
import os
from rust_doc import guide_title
import toc_rules
from types import to_dash_type

log = logging.getLogger('docset.rust')


# Creates a full qualified name based on prefix and name
def make_fqn(prefix, name):
    return "%s::%s" % (prefix, name)


# Appends #ref to path if not None
def path_with_ref(path, ref):
    if ref:
        return path + "#" + ref
    else:
        return path


@cached_html
def add_guide(ctx, tree):
    """
    This one should work only for nigthly builds

    Special case, main index "index.html" and "rustdoc.html"
    have the same title "Rust Documentation", which is rather
    confusing.
    """
    if ctx['rel_path'] == 'index.html':
        titles = ["Overview"]
    else:
        titles = scrape(tree, index_rules.guide_titles())

    if len(titles) > 0:
        ctx['idx'].add(guide_title(titles[0][0]),
                       "gd", ctx['rel_path'], to_dash_type)


@cached_html
def add_module(ctx, tree):
    fqn_prefix = os.path.dirname(ctx['rel_path']).replace(os.sep, "::")
    ctx['idx'].add(fqn_prefix, "mod", ctx['rel_path'], to_dash_type)
    update_toc(ctx, tree, toc_rules.by_type("mod"),
               lambda x: to_dash_type(x, x))


@cached_html
def add_decl_html(ctx, tree):
    idx = ctx['idx']

    rel_path = ctx['rel_path']
    name, _ = os.path.splitext(os.path.basename(rel_path))
    dirname = os.path.dirname(rel_path)

    fqn_prefix = dirname.replace(os.sep, "::")
    name_parts = name.split('.')
    if len(name_parts) < 2:
        log.error("Unexpected HTML file: %s" % rel_path)
        return
    ty, name = name_parts
    fqn = make_fqn(fqn_prefix, name)

{'fraction_non_alphanumeric': 0.1291542826263172, 'fraction_numerical': 0.0021615779519048904, 'mean_word_length': 3.274826789838337, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6681092', 'n_tokens_mistral': 1308, 'n_tokens_neox': 1255, 'n_words': 326}