<commit_before># -*- coding: utf-8 -*-
from __future__ import with_statement

import logging
import os.path
import shutil

import jinja2
import markdown

from ext import render_markdown, MarkdownJinja
from util import (
    git_clone, git_checkout,
    list_doc_tree, list_refs,
    tmp_mk, tmp_close, tmp_cp,
    get_git_tag, get_git_branch,
    parse_package_json, walk_files, uri_parse
)

__major__ = 0
__minor__ = 1
__micro__ = 0
__version__ = "{0}.{1}.{2}".format(__major__, __minor__, __micro__)

logger = logging.getLogger(__name__)


class Docfu(object):
    """
    A class which converts Markdown files in one git directory to HTML files in
    another. Made for documentation generation, this class also provides global
    template variables and processing."""

    def __init__(self, uri, root, dest, **kwargs):
        """ Docfu initialzation. """
        self.uri = uri_parse(uri)
        self.root = os.path.normpath(root)
        dest = os.path.abspath(os.path.expanduser(dest))

        self.template_globals = {}
        if 'template_globals' in kwargs:
            self.template_globals = kwargs['template_globals']

        self.base_template = 'base.html'
        if 'base_template' in kwargs:
            self.base_template = kwargs['base_template']

        if self.uri.startswith('file://'):
            self.uri = self.uri.replace("file://", "")
            self.repository_dir = os.path.expanduser(self.uri)
            self.git_repo = False
        else:
            self.repository_dir = git_clone(self.uri)
            self.git_repo = True

        source_src_dir = self.root
        if 'source_dir' in kwargs:
            source_src_dir = kwargs['source_dir']

        assets_src_dir = os.path.join(self.root, '_static')
        if 'assets_dir' in kwargs:
            assets_src_dir = kwargs['assets_dir']

        templates_src_dir = os.path.join(self.root, '_templates')
        if 'templates_dir' in kwargs:
            templates_src_dir = kwargs['templates_dir']

        self.source_src_dir = os.path.join(self.repository_dir, source_src_dir)
        self.assets_src_dir = os.path.join(self.repository_dir, assets_src_dir)

        self.templates_src_dir = os.path.join(self.repository_dir,
            templates_src_dir)
        branch = kwargs['branch'] if 'branch' in kwargs else None
        tag = kwargs['tag'] if 'tag' in kwargs else None

        self.branch = ''
        self.tag = ''
        if branch and self.git_repo:
            self.git_ref_type = 'branch'
            self.git_ref_val = branch
            self.branch = branch

        elif tag and self.git_repo:
            self.git_ref_type = 'tag'
            self.git_ref_val = tag
            self.tag = tag
        else:
            self.git_ref_type = 'file'
            self.git_ref_val = os.path.basename(self.uri)

        if self.git_repo:
            git_checkout(self.repository_dir, self.git_ref_type,
                self.git_ref_val)

        if "/" in self.git_ref_val:
            self.git_ref_val = self.git_ref_val.replace("/", "_")

        self.dest_root = dest
        self.dest = os.path.join(dest, self.git_ref_type, self.git_ref_val)
        self.source_dest_dir = os.path.join(self.dest)
        self.assets_dest_dir = os.path.join(self.dest, '_static')
        self._init_directories()

        self.template_globals = self._init_template_globals()

        logger.debug("""> Docfu
uri: %s
root: %s
dest: %s

>> Source
source: %s
static: %s
templates: %s

>> Dest
dest: %s
static: %s

>> Git ref
type: %s
value: %s """ % (self.uri, self.root, self.dest,
    self.source_src_dir, self.assets_src_dir, self.templates_src_dir,
    self.source_dest_dir, self.assets_dest_dir,
    self.template_globals['GIT_REF_TYPE'],
    self.template_globals['GIT_REF']))

        logger.debug(self.template_globals)

        self.source_files = walk_files(self.source_src_dir)
        self._env = self._init_template_engine()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        logger.info("Cleaning up ...")
        tmp_close(self.repository_dir)

    def __call__(self):
        self.render()
<commit_msg>feat(__init__.py): Do not delete local repository.<commit_after># -*- coding: utf-8 -*-
from __future__ import with_statement

import logging
import os.path
import shutil

import jinja2
import markdown

from ext import render_markdown, MarkdownJinja
from util import (
    git_clone, git_checkout,
    list_doc_tree, list_refs,
    tmp_mk, tmp_close, tmp_cp,
    get_git_tag, get_git_branch,
    parse_package_json, walk_files, uri_parse
)

__major__ = 0
__minor__ = 1
__micro__ = 0
__version__ = "{0}.{1}.{2}".format(__major__, __minor__, __micro__)

logger = logging.getLogger(__name__)


class Docfu(object):
    """
    A class which converts Markdown files in one git directory to HTML files in
    another. Made for documentation generation, this class also provides global
    template variables and processing."""

    def __init__(self, uri, root, dest, **kwargs):
        """ Docfu initialzation. """
        self.uri = uri_parse(uri)
        self.root = os.path.normpath(root)
        dest = os.path.abspath(os.path.expanduser(dest))

        self.template_globals = {}
        if 'template_globals' in kwargs:
            self.template_globals = kwargs['template_globals']

        self.base_template = 'base.html'
        if 'base_template' in kwargs:
            self.base_template = kwargs['base_template']

        if self.uri.startswith('file://'):
            self.uri = self.uri.replace("file://", "")
            self.repository_dir = os.path.expanduser(self.uri)
            self.git_repo = False
        else:
            self.repository_dir = git_clone(self.uri)
            self.git_repo = True

        source_src_dir = self.root
        if 'source_dir' in kwargs:
            source_src_dir = kwargs['source_dir']

        assets_src_dir = os.path.join(self.root, '_static')
        if 'assets_dir' in kwargs:
            assets_src_dir = kwargs['assets_dir']

        templates_src_dir = os.path.join(self.root, '_templates')
        if 'templates_dir' in kwargs:
            templates_src_dir = kwargs['templates_dir']

        self.source_src_dir = os.path.join(self.repository_dir, source_src_dir)
        self.assets_src_dir = os.path.join(self.repository_dir, assets_src_dir)

        self.templates_src_dir = os.path.join(self.repository_dir,
            templates_src_dir)
        branch = kwargs['branch'] if 'branch' in kwargs else None
        tag = kwargs['tag'] if 'tag' in kwargs else None

        self.branch = ''
        self.tag = ''
        if branch and self.git_repo:
            self.git_ref_type = 'branch'
            self.git_ref_val = branch
            self.branch = branch

        elif tag and self.git_repo:
            self.git_ref_type = 'tag'
            self.git_ref_val = tag
            self.tag = tag
        else:
            self.git_ref_type = 'file'
            self.git_ref_val = os.path.basename(self.uri)

        if self.git_repo:
            git_checkout(self.repository_dir, self.git_ref_type,
                self.git_ref_val)

        if "/" in self.git_ref_val:
            self.git_ref_val = self.git_ref_val.replace("/", "_")

        self.dest_root = dest
        self.dest = os.path.join(dest, self.git_ref_type, self.git_ref_val)
        self.source_dest_dir = os.path.join(self.dest)
        self.assets_dest_dir = os.path.join(self.dest, '_static')
        self._init_directories()

        self.template_globals = self._init_template_globals()

        logger.debug("""> Docfu
uri: %s
root: %s
dest: %s

>> Source
source: %s
static: %s
templates: %s

>> Dest
dest: %s
static: %s

>> Git ref
type: %s
value: %s """ % (self.uri, self.root, self.dest,
    self.source_src_dir, self.assets_src_dir, self.templates_src_dir,
    self.source_dest_dir, self.assets_dest_dir,
    self.template_globals['GIT_REF_TYPE'],
    self.template_globals['GIT_REF']))

        logger.debug(self.template_globals)

        self.source_files = walk_files(self.source_src_dir)
        self._env = self._init_template_engine()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        logger.info("Cleaning up ...")
        if self.git_repo:
            tmp_close(self.repository_dir)

    def __call__(self):
        self.render()

{'fraction_non_alphanumeric': 0.10928766797478892, 'fraction_numerical': 0.001902723272684029, 'mean_word_length': 2.54702657106706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1021173', 'n_tokens_mistral': 2728, 'n_tokens_neox': 2683, 'n_words': 565}