<commit_before>import os
import logging
import json
import yaml

from django.conf import settings
from django.template import Context, loader as template_loader

from readthedocs.doc_builder.base import BaseBuilder, restoring_chdir

log = logging.getLogger(__name__)

TEMPLATE_DIR = '%s/readthedocs/templates/mkdocs/readthedocs' % settings.SITE_ROOT
OVERRIDE_TEMPLATE_DIR = '%s/readthedocs/templates/mkdocs/overrides' % settings.SITE_ROOT


class BaseMkdocs(BaseBuilder):

    """
    Mkdocs builder
    """

    def __init__(self, *args, **kwargs):
        super(BaseMkdocs, self).__init__(*args, **kwargs)
        self.old_artifact_path = os.path.join(
            self.version.project.checkout_path(self.version.slug),
            self.build_dir)

    def append_conf(self, **kwargs):
        """
        Set mkdocs config values
        """

        # Pull mkdocs config data
        try:
            user_config = yaml.safe_load(open('mkdocs.yml', 'r'))
        except IOError:
            user_config = {
                'site_name': self.version.project.name,
            }

        # Handle custom docs dirs

        docs_dir = self.docs_dir(docs_dir=user_config.get('docs_dir'))
        self.create_index(extension='md')
        user_config['docs_dir'] = docs_dir

        # Set mkdocs config values

        media_url = getattr(settings, 'MEDIA_URL', 'https://media.readthedocs.org')

        # Mkdocs needs a full domain here because it tries to link to local media files
        if not media_url.startswith('http'):
            media_url = 'http://localhost:8000' + media_url

        if 'extra_javascript' in user_config:
            user_config['extra_javascript'].append('readthedocs-data.js')
            user_config['extra_javascript'].append(
                'readthedocs-dynamic-include.js')
            user_config['extra_javascript'].append(
                '%sjavascript/readthedocs-doc-embed.js' % media_url)
        else:
            user_config['extra_javascript'] = [
                'readthedocs-data.js',
                'readthedocs-dynamic-include.js',
                '%sjavascript/readthedocs-doc-embed.js' % media_url,
            ]

        if 'extra_css' in user_config:
            user_config['extra_css'].append(
                '%s/css/badge_only.css' % media_url)
            user_config['extra_css'].append(
                '%s/css/readthedocs-doc-embed.css' % media_url)
        else:
            user_config['extra_css'] = [
                '%scss/badge_only.css' % media_url,
                '%scss/readthedocs-doc-embed.css' % media_url,
            ]

        # Set our custom theme dir for mkdocs
        if 'theme_dir' not in user_config:
            user_config['theme_dir'] = TEMPLATE_DIR

        yaml.dump(user_config, open('mkdocs.yml', 'w'))

        # RTD javascript writing

        # Will be available in the JavaScript as READTHEDOCS_DATA.
        readthedocs_data = {
            'project': self.version.project.slug,
            'version': self.version.verbose_name,
            'language': self.version.project.language,
            'page': None,
            'theme': "readthedocs",
            'builder': "mkdocs",
            'docroot': docs_dir,
            'source_suffix': ".md",
            'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
            'commit': self.version.project.vcs_repo(self.version.slug).commit,
        }
        data_json = json.dumps(readthedocs_data, indent=4)
        data_ctx = Context({
            'data_json': data_json,
            'current_version': readthedocs_data['version'],
            'slug': readthedocs_data['project'],
            'html_theme': readthedocs_data['theme'],
            'pagename': None,
        })
        data_string = template_loader.get_template(
            'doc_builder/data.js.tmpl'
        ).render(data_ctx)

        data_file = open(os.path.join(docs_dir, 'readthedocs-data.js'), 'w+')
        data_file.write(data_string)
        data_file.write('\nREADTHEDOCS_DATA["page"] = mkdocs_page_name')
        data_file.close()

        include_ctx = Context({
            'global_analytics_code': getattr(settings, 'GLOBAL_ANALYTICS_CODE', 'UA-17997319-1'),
            'user_analytics_code': self.version.project.analytics_code,
        })
        include_string = template_loader.get_template(
            'doc_builder/include.js.tmpl'
        ).render(include_ctx)
        include_file = open(os.path.join(docs_dir, 'readthedocs-dynamic-include.js'), 'w+')
        include_file.write(include_string)
        include_file.close()

    def build(self, **kwargs):
        checkout_path = self.version.project.checkout_path(self.version.slug)
        # TODO does this have the self.project?
        # TODO --theme fails on the JSON builder
        build_command = [
            self.project.venv_bin(version=self.version.slug, bin='mkdocs'),
            self.builder,
            '--clean',
            '--site-dir', self.build_dir,
            '--theme', 'readthedocs',
        ]
        self.run(*build_command, cwd=checkout_path)


class MkdocsHTML(BaseMkdocs):
    type = 'mkdocs'
    builder = 'build'
    build_dir = '_build/html'


class MkdocsJSON(BaseMkdocs):
    type = 'mkdocs_json'
    builder = 'json'
    build_dir = '_build/json'<commit_msg>Correct call on mkdocs json and some cleanup on mkdocs backend<commit_after>import os
import logging
import json
import yaml

from django.conf import settings
from django.template import Context, loader as template_loader

from readthedocs.doc_builder.base import BaseBuilder, restoring_chdir

log = logging.getLogger(__name__)

TEMPLATE_DIR = '%s/readthedocs/templates/mkdocs/readthedocs' % settings.SITE_ROOT
OVERRIDE_TEMPLATE_DIR = '%s/readthedocs/templates/mkdocs/overrides' % settings.SITE_ROOT


class BaseMkdocs(BaseBuilder):

    """
    Mkdocs builder
    """

    def __init__(self, *args, **kwargs):
        super(BaseMkdocs, self).__init__(*args, **kwargs)
        self.old_artifact_path = os.path.join(
            self.version.project.checkout_path(self.version.slug),
            self.build_dir)

    def append_conf(self, **kwargs):
        """
        Set mkdocs config values
        """

        # Pull mkdocs config data
        try:
            user_config = yaml.safe_load(open('mkdocs.yml', 'r'))
        except IOError:
            user_config = {
                'site_name': self.version.project.name,
            }

        # Handle custom docs dirs

        docs_dir = self.docs_dir(docs_dir=user_config.get('docs_dir'))
        self.create_index(extension='md')
        user_config['docs_dir'] = docs_dir

        # Set mkdocs config values

        media_url = getattr(settings, 'MEDIA_URL', 'https://media.readthedocs.org')

        # Mkdocs needs a full domain here because it tries to link to local media files
        if not media_url.startswith('http'):
            media_url = 'http://localhost:8000' + media_url

        if 'extra_javascript' in user_config:
            user_config['extra_javascript'].append('readthedocs-data.js')
            user_config['extra_javascript'].append(
                'readthedocs-dynamic-include.js')
            user_config['extra_javascript'].append(
                '%sjavascript/readthedocs-doc-embed.js' % media_url)
        else:
            user_config['extra_javascript'] = [
                'readthedocs-data.js',
                'readthedocs-dynamic-include.js',
                '%sjavascript/readthedocs-doc-embed.js' % media_url,
            ]

        if 'extra_css' in user_config:
            user_config['extra_css'].append(
                '%s/css/badge_only.css' % media_url)
            user_config['extra_css'].append(
                '%s/css/readthedocs-doc-embed.css' % media_url)
        else:
            user_config['extra_css'] = [
                '%scss/badge_only.css' % media_url,
                '%scss/readthedocs-doc-embed.css' % media_url,
            ]

        # Set our custom theme dir for mkdocs
        if 'theme_dir' not in user_config:
            user_config['theme_dir'] = TEMPLATE_DIR

        yaml.dump(user_config, open('mkdocs.yml', 'w'))

        # RTD javascript writing

        # Will be available in the JavaScript as READTHEDOCS_DATA.
        readthedocs_data = {
            'project': self.version.project.slug,
            'version': self.version.verbose_name,
            'language': self.version.project.language,
            'page': None,
            'theme': "readthedocs",
            'builder': "mkdocs",
            'docroot': docs_dir,
            'source_suffix': ".md",
            'api_host': getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
            'commit': self.version.project.vcs_repo(self.version.slug).commit,
        }
        data_json = json.dumps(readthedocs_data, indent=4)
        data_ctx = Context({
            'data_json': data_json,
            'current_version': readthedocs_data['version'],
            'slug': readthedocs_data['project'],
            'html_theme': readthedocs_data['theme'],
            'pagename': None,
        })
        data_string = template_loader.get_template(
            'doc_builder/data.js.tmpl'
        ).render(data_ctx)

        data_file = open(os.path.join(docs_dir, 'readthedocs-data.js'), 'w+')
        data_file.write(data_string)
        data_file.write('\nREADTHEDOCS_DATA["page"] = mkdocs_page_name')
        data_file.close()

        include_ctx = Context({
            'global_analytics_code': getattr(settings, 'GLOBAL_ANALYTICS_CODE', 'UA-17997319-1'),
            'user_analytics_code': self.version.project.analytics_code,
        })
        include_string = template_loader.get_template(
            'doc_builder/include.js.tmpl'
        ).render(include_ctx)
        include_file = open(os.path.join(docs_dir, 'readthedocs-dynamic-include.js'), 'w+')
        include_file.write(include_string)
        include_file.close()

    def build(self, **kwargs):
        checkout_path = self.project.checkout_path(self.version.slug)
        build_command = [
            self.project.venv_bin(version=self.version.slug, bin='mkdocs'),
            self.builder,
            '--clean',
            '--site-dir', self.build_dir,
            '--theme', 'readthedocs',
        ]
        self.run(*build_command, cwd=checkout_path)


class MkdocsHTML(BaseMkdocs):
    type = 'mkdocs'
    builder = 'build'
    build_dir = '_build/html'


class MkdocsJSON(BaseMkdocs):
    type = 'mkdocs_json'
    builder = 'json'
    build_dir = '_build/json'

    def build(self, **kwargs):
        checkout_path = self.project.checkout_path(self.version.slug)
        build_command = [
            self.project.venv_bin(version=self.version.slug, bin='mkdocs'),
            self.builder,
            '--clean',
            '--site-dir', self.build_dir
        ]
        self.run(*build_command, cwd=checkout_path)
{'fraction_non_alphanumeric': 0.11925681859784001, 'fraction_numerical': 0.002562694490206846, 'mean_word_length': 2.3122158229766594, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4772803', 'n_tokens_mistral': 3481, 'n_tokens_neox': 3362, 'n_words': 566}