<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Loyc;
using Loyc.Collections;
using Loyc.Syntax;
using Loyc.Threading;
using LeMP.Prelude;
using System.Diagnostics;
using Loyc.Collections.Impl;

/// <summary>The lexical macro processor. Main classes: <see cref="LeMP.Compiler"/> and <see cref="LeMP.MacroProcessor"/>.</summary>
namespace LeMP
{
	/// <summary>
	/// For LeMP: an input file plus per-file options (input and output language) and output code.
	/// </summary>
	public class InputOutput
	{
		public InputOutput(ICharSource text, string fileName, IParsingService input = null, ILNodePrinter outPrinter = null, string outFileName = null)
		{
			Text = text; FileName = fileName ?? ""; InputLang = input; OutPrinter = outPrinter; OutFileName = outFileName;
		}
		public readonly ICharSource Text;
		public readonly string FileName;
		public IParsingService InputLang;
		public bool? PreserveComments; // null means unassigned (to use the Compiler default)
		public ParsingMode ParsingMode; // inputType argument when parsing with IParsingService.Parse
		public ILNodePrinter OutPrinter;
		public ILNodePrinterOptions OutOptions;
		public string OutFileName;
		public VList<LNode> Output;
		public override string ToString()
		{
			return FileName;
		}
	}

	/// <summary>
	/// Encapsulates the LeMP engine, a simple LISP-style macro processor, 
	/// suitable for running LLLPG and other lexical macros.
	/// </summary>
	/// <remarks>
	/// MacroProcessor itself only cares about a few nodes including 
	/// #importMacros and #unimportMacros, and { braces } (for scoping the 
	/// #import statements). The macro processor should be configured with any 
	/// needed macros like this:
	/// <code>
	///   var prelude = typeof(LeMP.Prelude.BuiltinMacros); // the default prelude
	///   var MP = new MacroProcessor(prelude, sink);
	///   MP.AddMacros(typeof(LeMP.StandardMacros).Assembly);
	///   MP.PreOpenedNamespaces.Add((Symbol) "LeMP.Prelude"); // already done for you
	///   MP.PreOpenedNamespaces.Add((Symbol) "LeMP");
	/// </code>
	/// In order for the input code to have access to macros, two steps are 
	/// necessary: you have to add the macro classes with <see cref="AddMacros"/>
	/// and then you have to import the namespace that contains the class(es).
	/// Higher-level code (e.g. <see cref="Compiler"/>) can define "always-open"
	/// namespaces by adding entries to PreOpenedNamespaces, and the code being 
	/// processed can open additional namespaces with a #importMacros(Namespace) 
	/// statement.
	/// <para/>
	/// MacroProcessor is not aware of any distinction between "statements"
	/// and "expressions"; it will run macros no matter where they are located,
	/// whether as standalone statements, attributes, or arguments to functions.
	/// <para/>
	/// MacroProcessor's main responsibilities are to keep track of a table of 
	/// registered macros (call <see cref="AddMacros"/> to register more), to
	/// keep track of which namespaces are open, to scan the input for macros to 
	/// call; and to control the printout of messages.
	/// <para/>
	/// This class processes a batch of files at once. Call either
	/// <see cref="ProcessSynchronously"/> or <see cref="ProcessParallel"/>.
	/// Parallelizing on a file-by-file basis is easy; each source file is completely 
	/// independent, since no semantic analysis is being done. 
	/// </remarks>
	public partial class MacroProcessor
	{
		IMessageSink _sink;
		public IMessageSink Sink { get { return _sink; } set { _sink = value; } }
		public int MaxExpansions = 255;

		[ThreadStatic]
		internal static MacroProcessor _current;
		/// <summary>Returns the <c>MacroProcessor</c> running on the current thread, or null if none.</summary>
		public static MacroProcessor Current { get { return _current; } }

		/// <summary>Initializes MacroProcessor with default prelude.</summary>
		public MacroProcessor(IMessageSink sink) : this(sink, typeof(BuiltinMacros)) { }

		/// <summary>Initializes MacroProcessor.</summary>
		/// <param name="sink">The destination for warning and error messages. NOTE: 
		/// this class can process files in parallel. Consider using a thread-safe
		/// implementation of <see cref="IMessageSink"/>.</param>
		/// <param name="prelude">An initial type from which to add macros.
		/// Omit this parameter to use typeof(LeMP.Prelude.BuiltinMacros).</param>
		public MacroProcessor(IMessageSink sink, Type prelude)
		{
			_sink = sink;
			if (prelude != null)
				AddMacros(prelude);
			AbortTimeout = TimeSpan.FromSeconds(30);
			PreOpenedNamespaces.Add(GSymbol.Empty);
			PreOpenedNamespaces.Add((Symbol)"LeMP.Prelude");
		}

		MMap<Symbol, InternalList<InternalMacroInfo>> _macros = new MMap<Symbol, InternalList<InternalMacroInfo>>();
		internal MMap<Symbol, InternalList<InternalMacroInfo>> Macros { get { return _macros; } }

		/// <summary>Macros in these namespaces will be available without an explicit 
		/// import command (#importMacros). By default this list has one item: 
		/// @@LeMP.Prelude (i.e. (Symbol)"LeMP.Prelude")</summary>
		public ICollection<Symbol> PreOpenedNamespaces { get { return _preOpenedNamespaces; } }
		internal MSet<Symbol> _preOpenedNamespaces = new MSet<Symbol>();

		/// <summary>Default values of scoped properties.</summary>
		/// <remarks>Code being processed can look up a scoped property named "N" with 
		/// <c>#getScopedProperty("N")</c> in LESv2 or EC#. This map is empty by default.
		/// Scoped properties are "scoped" in the sense that setting a property with 
		/// <c>#setScopedProperty(keyLiteral, valueLiteral)</c> takes effect only until 
		/// the end of the braced block in which it appears.
		/// <para/>
		/// The @@#inputFolder and @@#inputFileName properties (note: @@ is EC# 
		/// syntax for <see cref="Symbol"/>) are not normally stored in this collection; 
		/// when you use <see cref="ProcessSynchronously"/> or <see cref="ProcessParallel"/>, 
		/// @@#inputFolder and @@#inputFileName are set according to the folder and 
		/// filename in <see cref="InputOutput.FileName"/>. However, @@#inputFolder 
		/// is not set if the filename has no folder component, so this collection 
		/// could be used to provide a value of @@#inputFolder in that case.
		/// </remarks>
		public MMap<object, object> DefaultScopedProperties = new MMap<object, object>();

		#region Adding macros from types (AddMacros())

		public bool AddMacros(Type type)
		{
			bool any = false;
			foreach (var info in LeMP.MacroInfo.GetMacros(type, _sink)) {
				any = true;
				AddMacro(_macros, info);
			}
			return any;
		}

		public bool AddMacros(Assembly assembly, bool writeToSink = true)
		{
			bool any = false;
			foreach (Type type in assembly.GetExportedTypes()) {
				if (!type.IsGenericTypeDefinition &&
					type.GetCustomAttributes(typeof(ContainsMacrosAttribute), true).Any())
				{
					if (writeToSink && Sink.IsEnabled(Severity.Verbose))
						Sink.Write(Severity.Verbose, assembly.GetName().Name, "Adding macros in type '{0}'", type);
					any = AddMacros(type) || any;
				}
			}
			if (!any && writeToSink)
				Sink.Warning(assembly, "No macros found");
			return any;
		}

		internal static void AddMacro(MMap<Symbol, InternalList<InternalMacroInfo>> macros, LeMP.MacroInfo newMacro)
		{
			if (newMacro?.Macro == null)
				return; // TODO: should we throw? log an error?
			foreach (string name_ in newMacro.Names) {
				var name = (Symbol)name_;
				var macrosWithThisName = macros[name, InternalList<InternalMacroInfo>.Empty];

				// Check if the same macro was added earlier, possibly in a different namespace
				foreach (var macro in macrosWithThisName)
				{
					if (macro.Macro.Equals(newMacro.Macro)) {
						if (!macro.Namespaces.Contains(newMacro.Namespace))
							macro.Namespaces.Add(newMacro.Namespace);
						goto continueOuter;
					}
				}
				// It's a new macro, add it
				macrosWithThisName.Add(new InternalMacroInfo {
					Attr = newMacro,
					Macro = newMacro.Macro,
					Namespaces = new InternalList<Symbol>(new Symbol[] { newMacro.Namespace })
				});
				macros[name] = macrosWithThisName;
				continueOuter:;
			}
		}

		#endregion<commit_msg>AddMacros() now returns number of macros added<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Loyc;
using Loyc.Collections;
using Loyc.Syntax;
using Loyc.Threading;
using LeMP.Prelude;
using System.Diagnostics;
using Loyc.Collections.Impl;

/// <summary>The lexical macro processor. Main classes: <see cref="LeMP.Compiler"/> and <see cref="LeMP.MacroProcessor"/>.</summary>
namespace LeMP
{
	/// <summary>
	/// For LeMP: an input file plus per-file options (input and output language) and output code.
	/// </summary>
	public class InputOutput
	{
		public InputOutput(ICharSource text, string fileName, IParsingService input = null, ILNodePrinter outPrinter = null, string outFileName = null)
		{
			Text = text; FileName = fileName ?? ""; InputLang = input; OutPrinter = outPrinter; OutFileName = outFileName;
		}
		public readonly ICharSource Text;
		public readonly string FileName;
		public IParsingService InputLang;
		public bool? PreserveComments; // null means unassigned (to use the Compiler default)
		public ParsingMode ParsingMode; // inputType argument when parsing with IParsingService.Parse
		public ILNodePrinter OutPrinter;
		public ILNodePrinterOptions OutOptions;
		public string OutFileName;
		public VList<LNode> Output;
		public override string ToString()
		{
			return FileName;
		}
	}

	/// <summary>
	/// Encapsulates the LeMP engine, a simple LISP-style macro processor, 
	/// suitable for running LLLPG and other lexical macros.
	/// </summary>
	/// <remarks>
	/// MacroProcessor itself only cares about a few nodes including 
	/// #importMacros and #unimportMacros, and { braces } (for scoping the 
	/// #import statements). The macro processor should be configured with any 
	/// needed macros like this:
	/// <code>
	///   var prelude = typeof(LeMP.Prelude.BuiltinMacros); // the default prelude
	///   var MP = new MacroProcessor(prelude, sink);
	///   MP.AddMacros(typeof(LeMP.StandardMacros).Assembly);
	///   MP.PreOpenedNamespaces.Add((Symbol) "LeMP.Prelude"); // already done for you
	///   MP.PreOpenedNamespaces.Add((Symbol) "LeMP");
	/// </code>
	/// In order for the input code to have access to macros, two steps are 
	/// necessary: you have to add the macro classes with <see cref="AddMacros"/>
	/// and then you have to import the namespace that contains the class(es).
	/// Higher-level code (e.g. <see cref="Compiler"/>) can define "always-open"
	/// namespaces by adding entries to PreOpenedNamespaces, and the code being 
	/// processed can open additional namespaces with a #importMacros(Namespace) 
	/// statement.
	/// <para/>
	/// MacroProcessor is not aware of any distinction between "statements"
	/// and "expressions"; it will run macros no matter where they are located,
	/// whether as standalone statements, attributes, or arguments to functions.
	/// <para/>
	/// MacroProcessor's main responsibilities are to keep track of a table of 
	/// registered macros (call <see cref="AddMacros"/> to register more), to
	/// keep track of which namespaces are open, to scan the input for macros to 
	/// call; and to control the printout of messages.
	/// <para/>
	/// This class processes a batch of files at once. Call either
	/// <see cref="ProcessSynchronously"/> or <see cref="ProcessParallel"/>.
	/// Parallelizing on a file-by-file basis is easy; each source file is completely 
	/// independent, since no semantic analysis is being done. 
	/// </remarks>
	public partial class MacroProcessor
	{
		IMessageSink _sink;
		public IMessageSink Sink { get { return _sink; } set { _sink = value; } }
		public int MaxExpansions = 255;

		[ThreadStatic]
		internal static MacroProcessor _current;
		/// <summary>Returns the <c>MacroProcessor</c> running on the current thread, or null if none.</summary>
		public static MacroProcessor Current { get { return _current; } }

		/// <summary>Initializes MacroProcessor with default prelude.</summary>
		public MacroProcessor(IMessageSink sink) : this(sink, typeof(BuiltinMacros)) { }

		/// <summary>Initializes MacroProcessor.</summary>
		/// <param name="sink">The destination for warning and error messages. NOTE: 
		/// this class can process files in parallel. Consider using a thread-safe
		/// implementation of <see cref="IMessageSink"/>.</param>
		/// <param name="prelude">An initial type from which to add macros.
		/// Omit this parameter to use typeof(LeMP.Prelude.BuiltinMacros).</param>
		public MacroProcessor(IMessageSink sink, Type prelude)
		{
			_sink = sink;
			if (prelude != null)
				AddMacros(prelude);
			AbortTimeout = TimeSpan.FromSeconds(30);
			PreOpenedNamespaces.Add(GSymbol.Empty);
			PreOpenedNamespaces.Add((Symbol)"LeMP.Prelude");
		}

		MMap<Symbol, InternalList<InternalMacroInfo>> _macros = new MMap<Symbol, InternalList<InternalMacroInfo>>();
		internal MMap<Symbol, InternalList<InternalMacroInfo>> Macros { get { return _macros; } }

		/// <summary>Macros in these namespaces will be available without an explicit 
		/// import command (#importMacros). By default this list has one item: 
		/// @@LeMP.Prelude (i.e. (Symbol)"LeMP.Prelude")</summary>
		public ICollection<Symbol> PreOpenedNamespaces { get { return _preOpenedNamespaces; } }
		internal MSet<Symbol> _preOpenedNamespaces = new MSet<Symbol>();

		/// <summary>Default values of scoped properties.</summary>
		/// <remarks>Code being processed can look up a scoped property named "N" with 
		/// <c>#getScopedProperty("N")</c> in LESv2 or EC#. This map is empty by default.
		/// Scoped properties are "scoped" in the sense that setting a property with 
		/// <c>#setScopedProperty(keyLiteral, valueLiteral)</c> takes effect only until 
		/// the end of the braced block in which it appears.
		/// <para/>
		/// The @@#inputFolder and @@#inputFileName properties (note: @@ is EC# 
		/// syntax for <see cref="Symbol"/>) are not normally stored in this collection; 
		/// when you use <see cref="ProcessSynchronously"/> or <see cref="ProcessParallel"/>, 
		/// @@#inputFolder and @@#inputFileName are set according to the folder and 
		/// filename in <see cref="InputOutput.FileName"/>. However, @@#inputFolder 
		/// is not set if the filename has no folder component, so this collection 
		/// could be used to provide a value of @@#inputFolder in that case.
		/// </remarks>
		public MMap<object, object> DefaultScopedProperties = new MMap<object, object>();

		#region Adding macros from types (AddMacros())

		public int AddMacros(Type type)
		{
			int found = 0;
			foreach (var info in LeMP.MacroInfo.GetMacros(type, _sink)) {
				found++;
				AddMacro(_macros, info);
			}
			return found;
		}

		public int AddMacros(Assembly assembly, bool writeToSink = true)
		{
			int found = 0;
			foreach (Type type in assembly.GetExportedTypes()) {
				if (!type.IsGenericTypeDefinition &&
					type.GetCustomAttributes(typeof(ContainsMacrosAttribute), true).Any())
				{
					if (writeToSink && Sink.IsEnabled(Severity.Verbose))
						Sink.Write(Severity.Verbose, assembly.GetName().Name, "Adding macros in type '{0}'", type);
					found += AddMacros(type);
				}
			}
			if (found == 0 && writeToSink)
				Sink.Warning(assembly, "No macros found");
			return found;
		}

		internal static void AddMacro(MMap<Symbol, InternalList<InternalMacroInfo>> macros, LeMP.MacroInfo newMacro)
		{
			if (newMacro?.Macro == null)
				return; // TODO: should we throw? log an error?
			foreach (string name_ in newMacro.Names) {
				var name = (Symbol)name_;
				var macrosWithThisName = macros[name, InternalList<InternalMacroInfo>.Empty];

				// Check if the same macro was added earlier, possibly in a different namespace
				foreach (var macro in macrosWithThisName)
				{
					if (macro.Macro.Equals(newMacro.Macro)) {
						if (!macro.Namespaces.Contains(newMacro.Namespace))
							macro.Namespaces.Add(newMacro.Namespace);
						goto continueOuter;
					}
				}
				// It's a new macro, add it
				macrosWithThisName.Add(new InternalMacroInfo {
					Attr = newMacro,
					Macro = newMacro.Macro,
					Namespaces = new InternalList<Symbol>(new Symbol[] { newMacro.Namespace })
				});
				macros[name] = macrosWithThisName;
				continueOuter:;
			}
		}

		#endregion
{'fraction_non_alphanumeric': 0.11680377449794338, 'fraction_numerical': 0.001028308734575369, 'mean_word_length': 3.6928753902923646, 'pattern_counts': {'":': 0, '<': 131, '<?xml version=': 0, '>': 131, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1432853', 'n_tokens_mistral': 5435, 'n_tokens_neox': 4884, 'n_words': 1632}