<commit_before>/*
 * Copyright 2014 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License
 */

package nl.ulso.magisto.converter;

import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import nl.ulso.magisto.io.FileSystemAccessor;
import org.pegdown.Extensions;
import org.pegdown.PegDownProcessor;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Writer;
import java.nio.file.Path;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static java.util.regex.Pattern.MULTILINE;

/**
 * Converts Markdown files to HTML.
 */
class MarkdownToHtmlFileConverter implements FileConverter {

    private static final String TEMPLATE_PATH = "/nl/ulso/magisto";
    private static final String DEFAULT_PAGE_TEMPLATE = "page_template.ftl";
    private static final String CUSTOM_PAGE_TEMPLATE = ".page.ftl";
    private static final Pattern TITLE_PATTERN = Pattern.compile("^#+ (.*)$", MULTILINE);
    static final Set<String> MARKDOWN_EXTENSIONS = new HashSet<>(Arrays.asList("md", "markdown", "mdown"));

    private final Template template;
    private final PegDownProcessor markdownProcessor;
    private final CustomLinkRenderer linkRenderer;

    MarkdownToHtmlFileConverter(FileSystemAccessor fileSystemAccessor, Path sourceRoot) {
        try {
            template = loadTemplate(fileSystemAccessor, sourceRoot);
        } catch (IOException e) {
            throw new RuntimeException("Could not load built-in template", e);
        }
        markdownProcessor = new PegDownProcessor(Extensions.ALL - Extensions.HARDWRAPS);
        linkRenderer = new CustomLinkRenderer();
    }

    Template loadTemplate(FileSystemAccessor fileSystemAccessor, Path sourceRoot) throws IOException {
        final Configuration configuration = new Configuration(Configuration.VERSION_2_3_21);
        configuration.setDefaultEncoding("UTF-8");
        configuration.setDateTimeFormat("long");
        configuration.setSharedVariable("link", new LocalLinkRewriteDirective());
        if (fileSystemAccessor.exists(sourceRoot.resolve(CUSTOM_PAGE_TEMPLATE))) {
            configuration.setTemplateLoader(new CustomTemplateLoader(fileSystemAccessor, sourceRoot));
            return configuration.getTemplate(CUSTOM_PAGE_TEMPLATE);
        } else {
            configuration.setClassForTemplateLoading(MarkdownToHtmlFileConverter.class, TEMPLATE_PATH);
            return configuration.getTemplate(DEFAULT_PAGE_TEMPLATE);
        }
    }

    @Override
    public boolean supports(Path path) {
        return MARKDOWN_EXTENSIONS.contains(getFileExtension(path));
    }

    private String getFileExtension(Path path) {
        final String fileName = path.getFileName().toString();
        final int position = fileName.lastIndexOf('.');
        if (position < 1) {
            return "";
        }
        if (position == fileName.length()) {
            return "";
        }
        return fileName.substring(position + 1).toLowerCase();
    }

    @Override
    public Path getConvertedFileName(Path path) {
        return path.resolveSibling(linkRenderer.resolveLink(path.getFileName().toString()));
    }

    @Override
    public void convert(FileSystemAccessor fileSystemAccessor, Path sourceRoot, Path targetRoot, Path path)
            throws IOException {
        Logger.getGlobal().log(Level.INFO, String.format("Converting '%s' from Markdown to HTML.", path));
        final Path targetFile = targetRoot.resolve(getConvertedFileName(path));
        try (final Writer writer = fileSystemAccessor.newBufferedWriterForTextFile(targetFile)) {
            final String markdownText = readTextFile(fileSystemAccessor, sourceRoot.resolve(path));
            final Map<String, Object> model = createPageModel(path, markdownText);
            template.process(model, writer);
        } catch (TemplateException e) {
            throw new RuntimeException("Error in built-in FreeMarker template", e);
        }
    }

    Map<String, Object> createPageModel(Path path, String markdownText) {<commit_msg>Made errors in the FreeMarker template non-breaking.<commit_after>/*
 * Copyright 2014 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License
 */

package nl.ulso.magisto.converter;

import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import nl.ulso.magisto.io.FileSystemAccessor;
import org.pegdown.Extensions;
import org.pegdown.PegDownProcessor;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Writer;
import java.nio.file.Path;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static java.util.regex.Pattern.MULTILINE;

/**
 * Converts Markdown files to HTML.
 */
class MarkdownToHtmlFileConverter implements FileConverter {

    private static final String TEMPLATE_PATH = "/nl/ulso/magisto";
    private static final String DEFAULT_PAGE_TEMPLATE = "page_template.ftl";
    private static final String CUSTOM_PAGE_TEMPLATE = ".page.ftl";
    private static final Pattern TITLE_PATTERN = Pattern.compile("^#+ (.*)$", MULTILINE);
    static final Set<String> MARKDOWN_EXTENSIONS = new HashSet<>(Arrays.asList("md", "markdown", "mdown"));

    private final Template template;
    private final PegDownProcessor markdownProcessor;
    private final CustomLinkRenderer linkRenderer;

    MarkdownToHtmlFileConverter(FileSystemAccessor fileSystemAccessor, Path sourceRoot) {
        try {
            template = loadTemplate(fileSystemAccessor, sourceRoot);
        } catch (IOException e) {
            throw new RuntimeException("Could not load built-in template", e);
        }
        markdownProcessor = new PegDownProcessor(Extensions.ALL - Extensions.HARDWRAPS);
        linkRenderer = new CustomLinkRenderer();
    }

    Template loadTemplate(FileSystemAccessor fileSystemAccessor, Path sourceRoot) throws IOException {
        final Configuration configuration = new Configuration(Configuration.VERSION_2_3_21);
        configuration.setDefaultEncoding("UTF-8");
        configuration.setDateTimeFormat("long");
        configuration.setSharedVariable("link", new LocalLinkRewriteDirective());
        if (fileSystemAccessor.exists(sourceRoot.resolve(CUSTOM_PAGE_TEMPLATE))) {
            configuration.setTemplateLoader(new CustomTemplateLoader(fileSystemAccessor, sourceRoot));
            return configuration.getTemplate(CUSTOM_PAGE_TEMPLATE);
        } else {
            configuration.setClassForTemplateLoading(MarkdownToHtmlFileConverter.class, TEMPLATE_PATH);
            return configuration.getTemplate(DEFAULT_PAGE_TEMPLATE);
        }
    }

    @Override
    public boolean supports(Path path) {
        return MARKDOWN_EXTENSIONS.contains(getFileExtension(path));
    }

    private String getFileExtension(Path path) {
        final String fileName = path.getFileName().toString();
        final int position = fileName.lastIndexOf('.');
        if (position < 1) {
            return "";
        }
        if (position == fileName.length()) {
            return "";
        }
        return fileName.substring(position + 1).toLowerCase();
    }

    @Override
    public Path getConvertedFileName(Path path) {
        return path.resolveSibling(linkRenderer.resolveLink(path.getFileName().toString()));
    }

    @Override
    public void convert(FileSystemAccessor fileSystemAccessor, Path sourceRoot, Path targetRoot, Path path)
            throws IOException {
        Logger.getGlobal().log(Level.INFO, String.format("Converting '%s' from Markdown to HTML.", path));
        final Path targetFile = targetRoot.resolve(getConvertedFileName(path));
        try (final Writer writer = fileSystemAccessor.newBufferedWriterForTextFile(targetFile)) {
            final String markdownText = readTextFile(fileSystemAccessor, sourceRoot.resolve(path));
            final Map<String, Object> model = createPageModel(path, markdownText);
            template.process(model, writer);
        } catch (TemplateException e) {
            Logger.getGlobal().log(Level.SEVERE, String.format("There was a problem in your custom page template. " +
                    "All converted pages are probably broken! The cause: %s", e.getMessage()), e);
        }
    }

    Map<String, Object> createPageModel(Path path, String markdownText) {
{'fraction_non_alphanumeric': 0.0916788626385114, 'fraction_numerical': 0.003136107045787163, 'mean_word_length': 3.705853418593212, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6080502', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2445, 'n_words': 665}