<commit_before># -*- coding: utf-8 -*-

import os
import operator

from markdoc.cache import DocumentCache


class Builder(object):
    
    """An object to handle all the parts of the wiki building process."""
    
    def __init__(self, config):
        self.config = config
        self.doc_cache = DocumentCache(base=os.path.join(self.config['meta']['root'], 'wiki'))
    
    def crumbs(self, filename):
        
        """
        Produce a breadcrumbs list for the given filename.
        
        The crumbs are calculated based on the wiki root and the absolute path
        to the current file.
        
        Examples
        --------
        
        Assuming a wiki root of `/a/b/c`:
        
        * `a/b/c/wiki/index.md` => `[('index', None)]`
        
        * `a/b/c/wiki/subdir/index.md` =>
          `[('index', '/'), ('subdir', None)]`
        
        * `a/b/c/wiki/subdir/file.md` =>
          `[('index', '/'), ('subdir', '/subdir/'), ('file', None)]
        
        """
        
        doc_root = os.path.join(self.config['meta']['root'], 'wiki')
        rel_path = os.path.relpath(filename, start=doc_root)
        rel_components = rel_path.split(os.path.sep)
        terminus = os.path.splitext(rel_components.pop())[0]
        
        if not rel_components:
            return [(terminus, None)]
        elif terminus == 'index':
            terminus = os.path.splitext(rel_components.pop())[0]
        
        crumbs = [('index', '/')]
        for component in rel_components:
            path = '%s%s/' % (crumbs[-1][1], component)
            crumbs.append((component, path))
        
        crumbs.append((terminus, None))
        return crumbs
    
    def walk(self):
        
        """
        Walk through the wiki, yielding info for each document.
        
        For each document encountered, a `(filename, crumbs)` tuple will be
        yielded.
        """
        
        wiki_dir = os.path.join(self.config['meta']['root'], 'wiki')
        
        for dirpath, subdirs, files in os.walk(wiki_dir):
            remove_hidden(subdirs); subdirs.sort()<commit_msg>Patched Builder.crumbs() to handle relative paths for crumbs.<commit_after># -*- coding: utf-8 -*-

import os
import operator

from markdoc.cache import DocumentCache


class Builder(object):
    
    """An object to handle all the parts of the wiki building process."""
    
    def __init__(self, config):
        self.config = config
        self.doc_cache = DocumentCache(base=os.path.join(self.config['meta']['root'], 'wiki'))
    
    def crumbs(self, path):
        
        """
        Produce a breadcrumbs list for the given filename.
        
        The crumbs are calculated based on the wiki root and the absolute path
        to the current file.
        
        Examples
        --------
        
        Assuming a wiki root of `/a/b/c`:
        
        * `a/b/c/wiki/index.md` => `[('index', None)]`
        
        * `a/b/c/wiki/subdir/index.md` =>
          `[('index', '/'), ('subdir', None)]`
        
        * `a/b/c/wiki/subdir/file.md` =>
          `[('index', '/'), ('subdir', '/subdir/'), ('file', None)]
        
        """
        
        if os.path.isabs(path):
            path = self.doc_cache.relative(path)
        
        rel_components = path.split(os.path.sep)
        terminus = os.path.splitext(rel_components.pop())[0]
        
        if not rel_components:
            return [(terminus, None)]
        elif terminus == 'index':
            terminus = os.path.splitext(rel_components.pop())[0]
        
        crumbs = [('index', '/')]
        for component in rel_components:
            path = '%s%s/' % (crumbs[-1][1], component)
            crumbs.append((component, path))
        
        crumbs.append((terminus, None))
        return crumbs
    
    def walk(self):
        
        """
        Walk through the wiki, yielding info for each document.
        
        For each document encountered, a `(filename, crumbs)` tuple will be
        yielded.
        """
        
        wiki_dir = os.path.join(self.config['meta']['root'], 'wiki')
        
        for dirpath, subdirs, files in os.walk(wiki_dir):
            remove_hidden(subdirs); subdirs.sort()
{'fraction_non_alphanumeric': 0.14353163361661944, 'fraction_numerical': 0.002360717658168083, 'mean_word_length': 1.7747216764898495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2935693', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1260, 'n_words': 306}