<commit_before>package com.ruleoftech.markdown.page.generator.plugin;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.codehaus.plexus.util.StringUtils;
import org.pegdown.Extensions;
import org.pegdown.PegDownProcessor;

/**
 * Creates a static html from markdown files.
 * 
 */
@Mojo(name = "generate")
public class MdPageGeneratorMojo extends AbstractMojo {

	@Parameter(property = "generate.defaultTitle")
	private String defaultTitle;

	@Parameter(property = "generate.inputDirectory", defaultValue = "${project.basedir}/src/main/resources/markdown/")
	private String inputDirectory;

	@Parameter(property = "generate.outputDirectory", defaultValue = "${project.build.directory}/html/")
	private String outputDirectory;

	@Parameter(property = "generate.headerHtmlFile")
	private String headerHtmlFile;

	@Parameter(property = "generate.footerHtmlFile")
	private String footerHtmlFile;

    @Parameter(property = "generate.recursiveInput", defaultValue = "false")
    private boolean recursiveInput;
    
	// Possible options
	// SMARTS: Beautifies apostrophes, ellipses ("..." and ". . .") and dashes ("--" and "---")
	// QUOTES: Beautifies single quotes, double quotes and double angle quotes (« and »)
	// SMARTYPANTS: Convenience extension enabling both, SMARTS and QUOTES, at once.
	// ABBREVIATIONS: Abbreviations in the way of PHP Markdown Extra.
	// HARDWRAPS: Alternative handling of newlines, see Github-flavoured-Markdown
	// AUTOLINKS: Plain (undelimited) autolinks the way Github-flavoured-Markdown implements them.
	// TABLES: Tables similar to MultiMarkdown (which is in turn like the PHP Markdown Extra tables, but with colspan support).
	// DEFINITION LISTS: Definition lists in the way of PHP Markdown Extra.
	// FENCED CODE BLOCKS: Fenced Code Blocks in the way of PHP Markdown Extra or Github-flavoured-Markdown.
	// HTML BLOCK SUPPRESSION: Suppresses the output of HTML blocks.
	// INLINE HTML SUPPRESSION: Suppresses the output of inline HTML elements.
	// WIKILINKS Support [[Wiki-style links]] with a customizable URL rendering logic.
	@Parameter(property = "generate.pegdownExtensions", defaultValue = "TABLES")
	private String pegdownExtensions;

	private enum EPegdownExtensions {
		NONE(0x00), SMARTS(0x01), QUOTES(0x02), SMARTYPANTS(EPegdownExtensions.SMARTS.getValue() + EPegdownExtensions.QUOTES.getValue()), ABBREVIATIONS(
				0x04), HARDWRAPS(0x08), AUTOLINKS(0x10), TABLES(0x20), DEFINITIONS(0x40), FENCED_CODE_BLOCKS(0x80), WIKILINKS(0x100), ALL(
				0x0000FFFF), SUPPRESS_HTML_BLOCKS(0x00010000), SUPPRESS_INLINE_HTML(0x00020000), SUPPRESS_ALL_HTML(0x00030000);

		private final int value;

		EPegdownExtensions(int value) {
			this.value = value;
		}

		public int getValue() {
			return value;
		}

	}

	/**
	 * Comma separated string of directories to be copied.
	 */
	@Parameter(property = "generate.copyDirectories")
	private String copyDirectories;

	protected List<MarkdownDTO> markdownDTOs = new ArrayList<MarkdownDTO>();

	/**
	 * Execute the maven plugin.
	 * 
	 * @throws MojoExecutionException
	 *             Something went wrong
	 */
	@Override
	public void execute() throws MojoExecutionException {
		if (StringUtils.isNotEmpty(copyDirectories)) {
			getLog().info("Copy files from directories");
			for (String dir : copyDirectories.split(",")) {
				copyFiles(inputDirectory + File.separator + dir, outputDirectory + File.separator + dir);
			}
		}

		getLog().info("Read Markdown files from input directory");
		boolean hasFiles = readFiles();

        if (hasFiles) {
            getLog().info("Process Pegdown extension options");
            int options = getPegdownExtensions(pegdownExtensions);

            getLog().info("Parse Markdown to HTML");
            processMarkdown(markdownDTOs, options);
        }
	}

	private int getPegdownExtensions(String extensions) {
		int options = 0;
		for (String ext : Arrays.asList(extensions.split("\\s*,\\s*"))) {
			try {
				if (!ext.isEmpty()) {
					Field f = Extensions.class.getField(ext);
					options |= f.getInt(null);
				}
			} catch (NoSuchFieldException e) {
				throw new IllegalArgumentException("No such extension: " + ext);
			} catch (IllegalAccessException e) {
				throw new IllegalArgumentException("Cannot read int value for extension " + ext + ": " + e, e);
			}
		}

		// getLog().info("Pegdown extension options = " + options);

		return options;

	}

	/**
	 * Read Markdown files from directory.
	 * 
     * @return boolean
     *             Is there files to read
	 * @throws MojoExecutionException
	 *             Unable to load file
	 */
	private boolean readFiles() throws MojoExecutionException {
		getLog().debug("Read files from: " + inputDirectory);

		try {
			File inputFile = new File(inputDirectory);
			if (!inputFile.exists()) {
				getLog().info("There is no input folder for the project. Skipping.");
				return false;
			}
			 
			// Reading just the markdown dir and sub dirs if recursive option set
			List<File> markdownFiles = getFilesAsArray(FileUtils.iterateFiles(new File(inputDirectory), new String[] { "md" }, recursiveInput));

			for (File file : markdownFiles) {
				getLog().debug("File getName() " + file.getName());
				getLog().debug("File getAbsolutePath() " + file.getAbsolutePath());
				getLog().debug("File getPath() " + file.getPath());

				MarkdownDTO dto = new MarkdownDTO();
				dto.markdownFile = file;

                // TODO Fix copy and include the folder structure
				if (!StringUtils.isNotEmpty(defaultTitle)) {
					List<String> raw = FileUtils.readLines(file);
					dto.title = getTitle(raw);
				} else {
					dto.title = defaultTitle;
				}

				File htmlFile = new File(outputDirectory + "/" + file.getName().replaceAll(".md", ".html"));
				dto.htmlFile = htmlFile;

				markdownDTOs.add(dto);
			}<commit_msg>Keep directory structure when recursiveInput is true<commit_after>package com.ruleoftech.markdown.page.generator.plugin;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.codehaus.plexus.util.StringUtils;
import org.pegdown.Extensions;
import org.pegdown.PegDownProcessor;

/**
 * Creates a static html from markdown files.
 * 
 */
@Mojo(name = "generate")
public class MdPageGeneratorMojo extends AbstractMojo {

	@Parameter(property = "generate.defaultTitle")
	private String defaultTitle;

	@Parameter(property = "generate.inputDirectory", defaultValue = "${project.basedir}/src/main/resources/markdown/")
	private String inputDirectory;

	@Parameter(property = "generate.outputDirectory", defaultValue = "${project.build.directory}/html/")
	private String outputDirectory;

	@Parameter(property = "generate.headerHtmlFile")
	private String headerHtmlFile;

	@Parameter(property = "generate.footerHtmlFile")
	private String footerHtmlFile;

    @Parameter(property = "generate.recursiveInput", defaultValue = "false")
    private boolean recursiveInput;
    
	// Possible options
	// SMARTS: Beautifies apostrophes, ellipses ("..." and ". . .") and dashes ("--" and "---")
	// QUOTES: Beautifies single quotes, double quotes and double angle quotes (« and »)
	// SMARTYPANTS: Convenience extension enabling both, SMARTS and QUOTES, at once.
	// ABBREVIATIONS: Abbreviations in the way of PHP Markdown Extra.
	// HARDWRAPS: Alternative handling of newlines, see Github-flavoured-Markdown
	// AUTOLINKS: Plain (undelimited) autolinks the way Github-flavoured-Markdown implements them.
	// TABLES: Tables similar to MultiMarkdown (which is in turn like the PHP Markdown Extra tables, but with colspan support).
	// DEFINITION LISTS: Definition lists in the way of PHP Markdown Extra.
	// FENCED CODE BLOCKS: Fenced Code Blocks in the way of PHP Markdown Extra or Github-flavoured-Markdown.
	// HTML BLOCK SUPPRESSION: Suppresses the output of HTML blocks.
	// INLINE HTML SUPPRESSION: Suppresses the output of inline HTML elements.
	// WIKILINKS Support [[Wiki-style links]] with a customizable URL rendering logic.
	@Parameter(property = "generate.pegdownExtensions", defaultValue = "TABLES")
	private String pegdownExtensions;

	private enum EPegdownExtensions {
		NONE(0x00), SMARTS(0x01), QUOTES(0x02), SMARTYPANTS(EPegdownExtensions.SMARTS.getValue() + EPegdownExtensions.QUOTES.getValue()), ABBREVIATIONS(
				0x04), HARDWRAPS(0x08), AUTOLINKS(0x10), TABLES(0x20), DEFINITIONS(0x40), FENCED_CODE_BLOCKS(0x80), WIKILINKS(0x100), ALL(
				0x0000FFFF), SUPPRESS_HTML_BLOCKS(0x00010000), SUPPRESS_INLINE_HTML(0x00020000), SUPPRESS_ALL_HTML(0x00030000);

		private final int value;

		EPegdownExtensions(int value) {
			this.value = value;
		}

		public int getValue() {
			return value;
		}

	}

	/**
	 * Comma separated string of directories to be copied.
	 */
	@Parameter(property = "generate.copyDirectories")
	private String copyDirectories;

	protected List<MarkdownDTO> markdownDTOs = new ArrayList<MarkdownDTO>();

	/**
	 * Execute the maven plugin.
	 * 
	 * @throws MojoExecutionException
	 *             Something went wrong
	 */
	@Override
	public void execute() throws MojoExecutionException {
		if (StringUtils.isNotEmpty(copyDirectories)) {
			getLog().info("Copy files from directories");
			for (String dir : copyDirectories.split(",")) {
				copyFiles(inputDirectory + File.separator + dir, outputDirectory + File.separator + dir);
			}
		}

		getLog().info("Read Markdown files from input directory");
		boolean hasFiles = readFiles();

        if (hasFiles) {
            getLog().info("Process Pegdown extension options");
            int options = getPegdownExtensions(pegdownExtensions);

            getLog().info("Parse Markdown to HTML");
            processMarkdown(markdownDTOs, options);
        }
	}

	private int getPegdownExtensions(String extensions) {
		int options = 0;
		for (String ext : Arrays.asList(extensions.split("\\s*,\\s*"))) {
			try {
				if (!ext.isEmpty()) {
					Field f = Extensions.class.getField(ext);
					options |= f.getInt(null);
				}
			} catch (NoSuchFieldException e) {
				throw new IllegalArgumentException("No such extension: " + ext);
			} catch (IllegalAccessException e) {
				throw new IllegalArgumentException("Cannot read int value for extension " + ext + ": " + e, e);
			}
		}

		// getLog().info("Pegdown extension options = " + options);

		return options;

	}

	/**
	 * Read Markdown files from directory.
	 * 
     * @return boolean
     *             Is there files to read
	 * @throws MojoExecutionException
	 *             Unable to load file
	 */
	private boolean readFiles() throws MojoExecutionException {
		getLog().debug("Read files from: " + inputDirectory);

		try {
			File inputFile = new File(inputDirectory);
			if (!inputFile.exists()) {
				getLog().info("There is no input folder for the project. Skipping.");
				return false;
			}
			 
			// Reading just the markdown dir and sub dirs if recursive option set
			List<File> markdownFiles = getFilesAsArray(FileUtils.iterateFiles(new File(inputDirectory), new String[] { "md" }, recursiveInput));

			for (File file : markdownFiles) {
				getLog().debug("File getName() " + file.getName());
				getLog().debug("File getAbsolutePath() " + file.getAbsolutePath());
				getLog().debug("File getPath() " + file.getPath());

				MarkdownDTO dto = new MarkdownDTO();
				dto.markdownFile = file;

				if (!StringUtils.isNotEmpty(defaultTitle)) {
					List<String> raw = FileUtils.readLines(file);
					dto.title = getTitle(raw);
				} else {
					dto.title = defaultTitle;
				}

                File htmlFile = new File(
                        recursiveInput
                                ? outputDirectory + "/" + file.getParentFile().getPath().substring(0, inputFile.getPath().length()) + "/" + file.getName().replaceAll(".md", ".html")
                                : outputDirectory + "/" + file.getName().replaceAll(".md", ".html")
                );
				dto.htmlFile = htmlFile;

				markdownDTOs.add(dto);
			}
{'fraction_non_alphanumeric': 0.11695397754230587, 'fraction_numerical': 0.010200854024988139, 'mean_word_length': 3.840030616150019, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '683545', 'n_tokens_mistral': 4192, 'n_tokens_neox': 3696, 'n_words': 979}