<commit_before>using System;
using System.CodeDom.Compiler;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web.Configuration;
using System.Web.Razor;

using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Text;
using Microsoft.CodeAnalysis.Emit;

namespace StackExchange.Precompilation
{
    class Compilation
    {
        private readonly CSharpCommandLineArguments _cscArgs;
        private readonly DirectoryInfo _currentDirectory;
        private readonly Encoding _encoding;
        private readonly WebConfigurationFileMap _configMap;
        private readonly List<Diagnostic> _diagnostics;

        private const string DiagnosticCategory = "StackExchange.Precompilation";
        private static DiagnosticDescriptor FailedToCreateModule =
            new DiagnosticDescriptor("SE001", "Failed to instantiate ICompileModule", "Failed to instantiate ICompileModule '{0}': {1}", DiagnosticCategory, DiagnosticSeverity.Error, true);
        private static DiagnosticDescriptor UnknownFileType =
            new DiagnosticDescriptor("SE002", "Unknown file type", "Unknown file type '{0}'", DiagnosticCategory, DiagnosticSeverity.Error, true);
        private static DiagnosticDescriptor ViewGenerationFailed =
            new DiagnosticDescriptor("SE003", "View generation failed", "View generation failed: {0}", DiagnosticCategory, DiagnosticSeverity.Error, true);
        private static DiagnosticDescriptor FailedParsingSourceTree =
            new DiagnosticDescriptor("SE004", "Failed parsing source tree", "Failed parasing source tree: {0}", DiagnosticCategory, DiagnosticSeverity.Error, true);


        public Compilation(CSharpCommandLineArguments cscArgs, DirectoryInfo currentDirectory, Encoding defaultEncoding = null)
        {
            _diagnostics = new List<Diagnostic>();
            _cscArgs = cscArgs;
            _currentDirectory = currentDirectory;
            _encoding = _cscArgs.Encoding ?? defaultEncoding ?? Encoding.UTF8;

            _configMap = new WebConfigurationFileMap { VirtualDirectories = { { "/", new VirtualDirectoryMapping(currentDirectory.FullName, true) } } };

            AppDomain.CurrentDomain.SetData("DataDirectory", Path.Combine(currentDirectory.FullName, "App_Data")); // HACK mocking ASP.NET's ~/App_Data aka. |DataDirectory|
        }

        public bool Run()
        {
            var references = SetupReferences();
            var sources = LoadSources(_cscArgs.SourceFiles.Select(x => x.Path).ToArray());

            var compilationModules = LoadModules().ToList();

            var compilation = CSharpCompilation.Create(
                options: _cscArgs.CompilationOptions,
                references: references,
                syntaxTrees: sources,
                assemblyName: _cscArgs.CompilationName);

            var context = new BeforeCompileContext()
            {
                Modules = compilationModules,
                Arguments = _cscArgs,
                Compilation = compilation,
                Diagnostics = _diagnostics,
                Resources = _cscArgs.ManifestResources.ToList()
            };

            foreach (var module in compilationModules)
            {
                module.BeforeCompile(context);
            }

            var emitResult = Emit(context);

            _diagnostics.ForEach(x => Console.WriteLine(x.ToString())); // strings only, since the Console.Out textwriter is another app domain...

            return emitResult.Success;
        }

        private IEnumerable<ICompileModule> LoadModules()
        {
            var compilationSection = PrecompilerSection.Current;
            if (compilationSection == null) yield break;

            foreach(var module in compilationSection.CompileModules.Cast<CompileModuleElement>())
            {
                ICompileModule compileModule = null;
                try
                {
                    var type = Type.GetType(module.Type, false);
                    compileModule = Activator.CreateInstance(type, true) as ICompileModule;
                }
                catch(Exception ex)
                {
                    _diagnostics.Add(Diagnostic.Create(
                        FailedToCreateModule,
                        Location.Create(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, new TextSpan(), new LinePositionSpan()),
                        module.Type,
                        ex.Message));
                }
                if (compileModule != null)<commit_msg>always write out any diagnostics we might have<commit_after>using System;
using System.CodeDom.Compiler;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web.Configuration;
using System.Web.Razor;

using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Text;
using Microsoft.CodeAnalysis.Emit;

namespace StackExchange.Precompilation
{
    class Compilation
    {
        private readonly CSharpCommandLineArguments _cscArgs;
        private readonly DirectoryInfo _currentDirectory;
        private readonly Encoding _encoding;
        private readonly WebConfigurationFileMap _configMap;
        private readonly List<Diagnostic> _diagnostics;

        private const string DiagnosticCategory = "StackExchange.Precompilation";
        private static DiagnosticDescriptor FailedToCreateModule =
            new DiagnosticDescriptor("SE001", "Failed to instantiate ICompileModule", "Failed to instantiate ICompileModule '{0}': {1}", DiagnosticCategory, DiagnosticSeverity.Error, true);
        private static DiagnosticDescriptor UnknownFileType =
            new DiagnosticDescriptor("SE002", "Unknown file type", "Unknown file type '{0}'", DiagnosticCategory, DiagnosticSeverity.Error, true);
        private static DiagnosticDescriptor ViewGenerationFailed =
            new DiagnosticDescriptor("SE003", "View generation failed", "View generation failed: {0}", DiagnosticCategory, DiagnosticSeverity.Error, true);
        private static DiagnosticDescriptor FailedParsingSourceTree =
            new DiagnosticDescriptor("SE004", "Failed parsing source tree", "Failed parasing source tree: {0}", DiagnosticCategory, DiagnosticSeverity.Error, true);


        public Compilation(CSharpCommandLineArguments cscArgs, DirectoryInfo currentDirectory, Encoding defaultEncoding = null)
        {
            _diagnostics = new List<Diagnostic>();
            _cscArgs = cscArgs;
            _currentDirectory = currentDirectory;
            _encoding = _cscArgs.Encoding ?? defaultEncoding ?? Encoding.UTF8;

            _configMap = new WebConfigurationFileMap { VirtualDirectories = { { "/", new VirtualDirectoryMapping(currentDirectory.FullName, true) } } };

            AppDomain.CurrentDomain.SetData("DataDirectory", Path.Combine(currentDirectory.FullName, "App_Data")); // HACK mocking ASP.NET's ~/App_Data aka. |DataDirectory|
        }

        public bool Run()
        {
            try
            {
                var references = SetupReferences();
                var sources = LoadSources(_cscArgs.SourceFiles.Select(x => x.Path).ToArray());

                var compilationModules = LoadModules().ToList();

                var compilation = CSharpCompilation.Create(
                    options: _cscArgs.CompilationOptions,
                    references: references,
                    syntaxTrees: sources,
                    assemblyName: _cscArgs.CompilationName);

                var context = new BeforeCompileContext()
                {
                    Modules = compilationModules,
                    Arguments = _cscArgs,
                    Compilation = compilation,
                    Diagnostics = _diagnostics,
                    Resources = _cscArgs.ManifestResources.ToList()
                };

                foreach (var module in compilationModules)
                {
                    module.BeforeCompile(context);
                }

                var emitResult = Emit(context);
                return emitResult.Success;
            }
            finally
            {
                _diagnostics.ForEach(x => Console.WriteLine(x.ToString())); // strings only, since the Console.Out textwriter is another app domain...
            }
        }

        private IEnumerable<ICompileModule> LoadModules()
        {
            var compilationSection = PrecompilerSection.Current;
            if (compilationSection == null) yield break;

            foreach(var module in compilationSection.CompileModules.Cast<CompileModuleElement>())
            {
                ICompileModule compileModule = null;
                try
                {
                    var type = Type.GetType(module.Type, false);
                    compileModule = Activator.CreateInstance(type, true) as ICompileModule;
                }
                catch(Exception ex)
                {
                    _diagnostics.Add(Diagnostic.Create(
                        FailedToCreateModule,
                        Location.Create(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, new TextSpan(), new LinePositionSpan()),
                        module.Type,
                        ex.Message));
                }
                if (compileModule != null)
{'fraction_non_alphanumeric': 0.07757449032932566, 'fraction_numerical': 0.0037637219027705174, 'mean_word_length': 2.3564912280701753, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6980192', 'n_tokens_mistral': 2296, 'n_tokens_neox': 2119, 'n_words': 540}