<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using Facility.Definition;
using Facility.Definition.CodeGen;
using Facility.Definition.Fsd;

namespace Facility.Console
{
	/// <summary>
	/// Base class for code generator console applications.
	/// </summary>
	public abstract class CodeGeneratorApp
	{
		/// <summary>
		/// Called to execute the code generator application.
		/// </summary>
		/// <param name="args">The command-line arguments.</param>
		/// <returns>The exit code.</returns>
		public int Run(IReadOnlyList<string> args)
		{
			try
			{
				var argsReader = new ArgsReader(args);
				if (argsReader.ReadHelpFlag())
				{
					foreach (string line in Description)
						System.Console.WriteLine(line);
					System.Console.WriteLine();
					WriteUsage();
					return 0;
				}

				var generator = CreateGenerator(argsReader);
				generator.GeneratorName = s_assemblyName;
				if (SupportsCustomIndent)
					generator.IndentText = argsReader.ReadIndentOption();
				if (SupportsCustomNewLine)
					generator.NewLine = argsReader.ReadNewLineOption();

				bool shouldClean = SupportsClean && argsReader.ReadCleanFlag();
				bool isQuiet = argsReader.ReadQuietFlag();
				bool isVerify = argsReader.ReadVerifyFlag();
				bool isDryRun = argsReader.ReadDryRunFlag();

				string inputPath = argsReader.ReadArgument();
				if (inputPath == null && !SupportsSingleOutput)
					throw new ArgsReaderException("Missing input path.");

				string outputPath = argsReader.ReadArgument();
				if (outputPath == null && !SupportsSingleOutput)
					throw new ArgsReaderException("Missing output path.");

				argsReader.VerifyComplete();

				NamedText input;
				if (inputPath == null || inputPath == "-")
				{
					input = new NamedText("", System.Console.In.ReadToEnd());
				}
				else
				{
					if (!File.Exists(inputPath))
						throw new ApplicationException("Input file does not exist: " + inputPath);
					input = new NamedText(Path.GetFileName(inputPath), File.ReadAllText(inputPath));
				}

				var service = new FsdParser().ParseDefinition(input);

				PrepareGenerator(generator, service, outputPath);
				var output = generator.GenerateOutput(service);

				if (SupportsSingleOutput && (outputPath == null ||
					(!outputPath.EndsWith("/", StringComparison.Ordinal) && !outputPath.EndsWith("\\", StringComparison.Ordinal) && !Directory.Exists(outputPath))))
				{
					if (output.NamedTexts.Count > 1)
						throw new InvalidOperationException("Multiple outputs not expected.");

					if (output.NamedTexts.Count == 1)
					{
						var namedText = output.NamedTexts[0];

						if (outputPath == null)
							System.Console.Write(namedText.Text);
						else if (ShouldWriteByteOrderMark(namedText.Name))
							File.WriteAllText(outputPath, namedText.Text, s_utf8WithBom);
						else
							File.WriteAllText(outputPath, namedText.Text);
					}
				}
				else
				{
					if (outputPath == null)
						throw new ArgsReaderException("Missing output path.");

					var namedTextsToWrite = new List<NamedText>();
					foreach (var namedText in output.NamedTexts)
					{
						string existingFilePath = Path.Combine(outputPath, namedText.Name);
						if (File.Exists(existingFilePath))
						{
							if (namedText.Text != File.ReadAllText(existingFilePath))
							{
								namedTextsToWrite.Add(namedText);
								if (!isQuiet)
									System.Console.WriteLine("changed " + namedText.Name);
							}
						}
						else
						{
							namedTextsToWrite.Add(namedText);
							if (!isQuiet)
								System.Console.WriteLine("added " + namedText.Name);
						}
					}

					var namesToDelete = new List<string>();
					if (shouldClean && output.PatternsToClean.Count != 0)
					{
						var directoryInfo = new DirectoryInfo(outputPath);
						if (directoryInfo.Exists)
						{
							foreach (string nameMatchingPattern in FindNamesMatchingPatterns(directoryInfo, output.PatternsToClean))
							{
								if (output.NamedTexts.All(x => x.Name != nameMatchingPattern))
								{
									namesToDelete.Add(nameMatchingPattern);
									if (!isQuiet)
										System.Console.WriteLine("removed " + nameMatchingPattern);
								}<commit_msg>Ignore CR when verifying files.<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using Facility.Definition;
using Facility.Definition.CodeGen;
using Facility.Definition.Fsd;

namespace Facility.Console
{
	/// <summary>
	/// Base class for code generator console applications.
	/// </summary>
	public abstract class CodeGeneratorApp
	{
		/// <summary>
		/// Called to execute the code generator application.
		/// </summary>
		/// <param name="args">The command-line arguments.</param>
		/// <returns>The exit code.</returns>
		public int Run(IReadOnlyList<string> args)
		{
			try
			{
				var argsReader = new ArgsReader(args);
				if (argsReader.ReadHelpFlag())
				{
					foreach (string line in Description)
						System.Console.WriteLine(line);
					System.Console.WriteLine();
					WriteUsage();
					return 0;
				}

				var generator = CreateGenerator(argsReader);
				generator.GeneratorName = s_assemblyName;
				if (SupportsCustomIndent)
					generator.IndentText = argsReader.ReadIndentOption();
				if (SupportsCustomNewLine)
					generator.NewLine = argsReader.ReadNewLineOption();

				bool shouldClean = SupportsClean && argsReader.ReadCleanFlag();
				bool isQuiet = argsReader.ReadQuietFlag();
				bool isVerify = argsReader.ReadVerifyFlag();
				bool isDryRun = argsReader.ReadDryRunFlag();

				string inputPath = argsReader.ReadArgument();
				if (inputPath == null && !SupportsSingleOutput)
					throw new ArgsReaderException("Missing input path.");

				string outputPath = argsReader.ReadArgument();
				if (outputPath == null && !SupportsSingleOutput)
					throw new ArgsReaderException("Missing output path.");

				argsReader.VerifyComplete();

				NamedText input;
				if (inputPath == null || inputPath == "-")
				{
					input = new NamedText("", System.Console.In.ReadToEnd());
				}
				else
				{
					if (!File.Exists(inputPath))
						throw new ApplicationException("Input file does not exist: " + inputPath);
					input = new NamedText(Path.GetFileName(inputPath), File.ReadAllText(inputPath));
				}

				var service = new FsdParser().ParseDefinition(input);

				PrepareGenerator(generator, service, outputPath);
				var output = generator.GenerateOutput(service);

				if (SupportsSingleOutput && (outputPath == null ||
					(!outputPath.EndsWith("/", StringComparison.Ordinal) && !outputPath.EndsWith("\\", StringComparison.Ordinal) && !Directory.Exists(outputPath))))
				{
					if (output.NamedTexts.Count > 1)
						throw new InvalidOperationException("Multiple outputs not expected.");

					if (output.NamedTexts.Count == 1)
					{
						var namedText = output.NamedTexts[0];

						if (outputPath == null)
							System.Console.Write(namedText.Text);
						else if (ShouldWriteByteOrderMark(namedText.Name))
							File.WriteAllText(outputPath, namedText.Text, s_utf8WithBom);
						else
							File.WriteAllText(outputPath, namedText.Text);
					}
				}
				else
				{
					if (outputPath == null)
						throw new ArgsReaderException("Missing output path.");

					var namedTextsToWrite = new List<NamedText>();
					foreach (var namedText in output.NamedTexts)
					{
						string existingFilePath = Path.Combine(outputPath, namedText.Name);
						if (File.Exists(existingFilePath))
						{
							// ignore CR when comparing files
							if (namedText.Text.Replace("\r", "") != File.ReadAllText(existingFilePath).Replace("\r", ""))
							{
								namedTextsToWrite.Add(namedText);
								if (!isQuiet)
									System.Console.WriteLine("changed " + namedText.Name);
							}
						}
						else
						{
							namedTextsToWrite.Add(namedText);
							if (!isQuiet)
								System.Console.WriteLine("added " + namedText.Name);
						}
					}

					var namesToDelete = new List<string>();
					if (shouldClean && output.PatternsToClean.Count != 0)
					{
						var directoryInfo = new DirectoryInfo(outputPath);
						if (directoryInfo.Exists)
						{
							foreach (string nameMatchingPattern in FindNamesMatchingPatterns(directoryInfo, output.PatternsToClean))
							{
								if (output.NamedTexts.All(x => x.Name != nameMatchingPattern))
								{
									namesToDelete.Add(nameMatchingPattern);
									if (!isQuiet)
										System.Console.WriteLine("removed " + nameMatchingPattern);
								}
{'fraction_non_alphanumeric': 0.11803698104430747, 'fraction_numerical': 0.0013955111059425515, 'mean_word_length': 3.176784847013113, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4029097', 'n_tokens_mistral': 3306, 'n_tokens_neox': 2421, 'n_words': 529}