<commit_before>package dyvilx.tools.compiler;

import dyvil.collection.List;
import dyvil.collection.Set;
import dyvil.collection.mutable.TreeSet;
import dyvil.io.Console;
import dyvil.io.FileUtils;
import dyvilx.tools.BasicTool;
import dyvilx.tools.compiler.ast.external.ExternalHeader;
import dyvilx.tools.compiler.ast.structure.Package;
import dyvilx.tools.compiler.ast.type.builtin.Types;
import dyvilx.tools.compiler.config.ArgumentParser;
import dyvilx.tools.compiler.config.CompilerConfig;
import dyvilx.tools.compiler.lang.I18n;
import dyvilx.tools.compiler.library.Library;
import dyvilx.tools.compiler.phase.ICompilerPhase;
import dyvilx.tools.compiler.sources.FileFinder;
import dyvilx.tools.compiler.util.TestThread;
import dyvilx.tools.compiler.util.Util;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;

import static dyvilx.tools.compiler.sources.DyvilFileType.*;

public class DyvilCompiler extends BasicTool
{
	public static final String VERSION         = "$$compilerVersion$$";
	public static final String DYVIL_VERSION   = "$$version$$";
	public static final String LIBRARY_VERSION = "$$libraryVersion$$";

	public final Set<ICompilerPhase> phases = new TreeSet<>();
	public final CompilerConfig      config = this.createConfig();

	public final FileFinder fileFinder = new FileFinder();

	protected CompilerConfig createConfig()
	{
		return new CompilerConfig(this);
	}

	@Override
	public int run(InputStream in, OutputStream out, OutputStream err, String... arguments)
	{
		final long startTime = System.nanoTime();

		this.initOutput(out, err);

		this.log(I18n.get("compiler.init", DyvilCompiler.VERSION, DyvilCompiler.DYVIL_VERSION));
		this.log("");

		final int exitCode = this.run(arguments);

		final long endTime = System.nanoTime();
		final boolean colors = this.config.useAnsiColors();

		final StringBuilder builder = new StringBuilder();

		if (exitCode != 0)
		{
			if (colors)
			{
				builder.append(Console.ANSI_RED);
			}
			builder.append(I18n.get("compilation.failure"));
		}
		else
		{
			if (colors)
			{
				builder.append(Console.ANSI_GREEN);
			}
			builder.append(I18n.get("compilation.success"));
		}
		if (colors)
		{
			builder.append(Console.ANSI_RESET);
		}

		builder.append(" (").append(Util.toTime(endTime - startTime)).append(')');

		this.log(builder.toString());
		return this.getExitCode();
	}

	private int run(String[] arguments)
	{
		if (!this.baseInit(arguments))
		{
			this.shutdown();
			return -1;
		}

		this.loadLibraries();
		this.findFiles();

		if (!this.applyPhases())
		{
			this.shutdown();
			return -1;
		}

		this.shutdown();
		return this.getExitCode();
	}

	public boolean baseInit(String[] args)
	{
		// Sets up States from arguments
		this.processArguments(args);

		final List<File> sourceDirs = this.config.sourceDirs;
		if (sourceDirs.isEmpty())
		{
			this.log(I18n.get("config.source_path.missing"));
			return false;
		}

		for (File file : sourceDirs)
		{
			if (file.exists())
			{
				return true;
			}
		}

		this.log(I18n.get("config.source_path.not_found", sourceDirs));
		return false;
	}

	public void processArguments(String[] args)
	{
		for (String arg : args)
		{
			this.processArgument(arg);
		}
	}

	public void processArgument(String arg)
	{
		ArgumentParser.parseArgument(arg, this);
	}

	public void loadLibraries()
	{
		final List<Library> libraries = this.config.libraries;

		// Make sure to add the dyvil and java libraries at the end
		libraries.add(Library.dyvilLibrary);
		libraries.add(Library.javaLibrary);

		final int libs = libraries.size();
		final long startTime = System.nanoTime();

		// Loads libraries
		for (Library library : libraries)
		{
			library.loadLibrary();
		}

		Package.initRoot(this);

		final long endTime = System.nanoTime();
		this.log(I18n.get("library.found", libs == 1 ? I18n.get("libraries.1") : I18n.get("libraries.n", libs),
		                  Util.toTime(endTime - startTime)));
	}

	protected void findFiles()
	{
		final long startTime = System.nanoTime();

		final List<File> sourceDirs = this.config.sourceDirs;
		final File outputDir = this.config.getOutputDir();

		this.log(I18n.get("compilation.init", sourceDirs, outputDir));

		// Scan for Packages and Compilation Units
		this.setupFileFinder();

		for (File sourceDir : sourceDirs)
		{
			this.fileFinder.process(this, sourceDir, outputDir, Package.rootPackage);
		}

		Package.init();

		final int fileCount = this.fileFinder.files.size();
		final int unitCount = this.fileFinder.units.size();

		final long endTime = System.nanoTime();

		this.log(I18n.get("files.found", fileCount == 1 ? I18n.get("files.1") : I18n.get("files.n", fileCount),
		                  unitCount == 1 ? I18n.get("units.1") : I18n.get("units.n", unitCount),
		                  Util.toTime(endTime - startTime)));
		this.log("");
	}

	protected void setupFileFinder()
	{
		this.fileFinder.registerFileType(DYVIL_EXTENSION, DYVIL_UNIT);
		this.fileFinder.registerFileType(".dyvil", DYVIL_UNIT); // legacy
		this.fileFinder.registerFileType(HEADER_EXTENSION, DYVIL_HEADER);
		this.fileFinder.registerFileType(".dyvilh", DYVIL_HEADER); // legacy
	}

	public boolean applyPhases()
	{
		final int phases = this.phases.size();

		// Apply states
		if (this.config.isDebug())
		{
			this.log(phases == 1 ?<commit_msg>Sort compilation units before applying phases<commit_after>package dyvilx.tools.compiler;

import dyvil.collection.List;
import dyvil.collection.Set;
import dyvil.collection.mutable.TreeSet;
import dyvil.io.Console;
import dyvil.io.FileUtils;
import dyvilx.tools.BasicTool;
import dyvilx.tools.compiler.ast.external.ExternalHeader;
import dyvilx.tools.compiler.ast.structure.Package;
import dyvilx.tools.compiler.ast.type.builtin.Types;
import dyvilx.tools.compiler.config.ArgumentParser;
import dyvilx.tools.compiler.config.CompilerConfig;
import dyvilx.tools.compiler.lang.I18n;
import dyvilx.tools.compiler.library.Library;
import dyvilx.tools.compiler.phase.ICompilerPhase;
import dyvilx.tools.compiler.sources.FileFinder;
import dyvilx.tools.compiler.util.TestThread;
import dyvilx.tools.compiler.util.Util;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;

import static dyvilx.tools.compiler.sources.DyvilFileType.*;

public class DyvilCompiler extends BasicTool
{
	public static final String VERSION         = "$$compilerVersion$$";
	public static final String DYVIL_VERSION   = "$$version$$";
	public static final String LIBRARY_VERSION = "$$libraryVersion$$";

	public final Set<ICompilerPhase> phases = new TreeSet<>();
	public final CompilerConfig      config = this.createConfig();

	public final FileFinder fileFinder = new FileFinder();

	protected CompilerConfig createConfig()
	{
		return new CompilerConfig(this);
	}

	@Override
	public int run(InputStream in, OutputStream out, OutputStream err, String... arguments)
	{
		final long startTime = System.nanoTime();

		this.initOutput(out, err);

		this.log(I18n.get("compiler.init", DyvilCompiler.VERSION, DyvilCompiler.DYVIL_VERSION));
		this.log("");

		final int exitCode = this.run(arguments);

		final long endTime = System.nanoTime();
		final boolean colors = this.config.useAnsiColors();

		final StringBuilder builder = new StringBuilder();

		if (exitCode != 0)
		{
			if (colors)
			{
				builder.append(Console.ANSI_RED);
			}
			builder.append(I18n.get("compilation.failure"));
		}
		else
		{
			if (colors)
			{
				builder.append(Console.ANSI_GREEN);
			}
			builder.append(I18n.get("compilation.success"));
		}
		if (colors)
		{
			builder.append(Console.ANSI_RESET);
		}

		builder.append(" (").append(Util.toTime(endTime - startTime)).append(')');

		this.log(builder.toString());
		return this.getExitCode();
	}

	private int run(String[] arguments)
	{
		if (!this.baseInit(arguments))
		{
			this.shutdown();
			return -1;
		}

		this.loadLibraries();
		this.findFiles();

		if (!this.applyPhases())
		{
			this.shutdown();
			return -1;
		}

		this.shutdown();
		return this.getExitCode();
	}

	public boolean baseInit(String[] args)
	{
		// Sets up States from arguments
		this.processArguments(args);

		final List<File> sourceDirs = this.config.sourceDirs;
		if (sourceDirs.isEmpty())
		{
			this.log(I18n.get("config.source_path.missing"));
			return false;
		}

		for (File file : sourceDirs)
		{
			if (file.exists())
			{
				return true;
			}
		}

		this.log(I18n.get("config.source_path.not_found", sourceDirs));
		return false;
	}

	public void processArguments(String[] args)
	{
		for (String arg : args)
		{
			this.processArgument(arg);
		}
	}

	public void processArgument(String arg)
	{
		ArgumentParser.parseArgument(arg, this);
	}

	public void loadLibraries()
	{
		final List<Library> libraries = this.config.libraries;

		// Make sure to add the dyvil and java libraries at the end
		libraries.add(Library.dyvilLibrary);
		libraries.add(Library.javaLibrary);

		final int libs = libraries.size();
		final long startTime = System.nanoTime();

		// Loads libraries
		for (Library library : libraries)
		{
			library.loadLibrary();
		}

		Package.initRoot(this);

		final long endTime = System.nanoTime();
		this.log(I18n.get("library.found", libs == 1 ? I18n.get("libraries.1") : I18n.get("libraries.n", libs),
		                  Util.toTime(endTime - startTime)));
	}

	protected void findFiles()
	{
		final long startTime = System.nanoTime();

		final List<File> sourceDirs = this.config.sourceDirs;
		final File outputDir = this.config.getOutputDir();

		this.log(I18n.get("compilation.init", sourceDirs, outputDir));

		// Scan for Packages and Compilation Units
		this.setupFileFinder();

		for (File sourceDir : sourceDirs)
		{
			this.fileFinder.process(this, sourceDir, outputDir, Package.rootPackage);
		}

		Package.init();

		// sort compilation units by case insensitive absolute path to ensure consistency across filesystems
		this.fileFinder.units.sort(
			Comparator.comparing(unit -> unit.getFileSource().file().getAbsolutePath(), String.CASE_INSENSITIVE_ORDER));

		final int fileCount = this.fileFinder.files.size();
		final int unitCount = this.fileFinder.units.size();

		final long endTime = System.nanoTime();

		this.log(I18n.get("files.found", fileCount == 1 ? I18n.get("files.1") : I18n.get("files.n", fileCount),
		                  unitCount == 1 ? I18n.get("units.1") : I18n.get("units.n", unitCount),
		                  Util.toTime(endTime - startTime)));
		this.log("");
	}

	protected void setupFileFinder()
	{
		this.fileFinder.registerFileType(DYVIL_EXTENSION, DYVIL_UNIT);
		this.fileFinder.registerFileType(".dyvil", DYVIL_UNIT); // legacy
		this.fileFinder.registerFileType(HEADER_EXTENSION, DYVIL_HEADER);
		this.fileFinder.registerFileType(".dyvilh", DYVIL_HEADER); // legacy
	}

	public boolean applyPhases()
	{
		final int phases = this.phases.size();

		// Apply states
		if (this.config.isDebug())
		{
			this.log(phases == 1 ?
{'fraction_non_alphanumeric': 0.13243094933235777, 'fraction_numerical': 0.007316627034936894, 'mean_word_length': 4.053142329020333, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2555498', 'n_tokens_mistral': 3979, 'n_tokens_neox': 3587, 'n_words': 722}