<commit_before>﻿using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using Dotnet.Script.DependencyModel.NuGet;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.CodeAnalysis.Scripting.Hosting;
using Microsoft.Extensions.Logging;
using OmniSharp.Helpers;
using OmniSharp.Roslyn.EditorConfig;
using OmniSharp.Roslyn.Utilities;

namespace OmniSharp.Script
{
    public class ScriptProjectProvider
    {
        private const string BinderFlagsType = "Microsoft.CodeAnalysis.CSharp.BinderFlags";
        private const string TopLevelBinderFlagsProperty = "TopLevelBinderFlags";
        private const string ReferencesSupersedeLowerVersionsProperty = "ReferencesSupersedeLowerVersions_internal_protected_set";
        private const string IgnoreCorLibraryDuplicatedTypesField = "IgnoreCorLibraryDuplicatedTypes";
        private const string RuntimeMetadataReferenceResolverType = "Microsoft.CodeAnalysis.Scripting.Hosting.RuntimeMetadataReferenceResolver";
        private const string ResolverField = "_resolver";
        private const string FileReferenceProviderField = "_fileReferenceProvider";

        // aligned with CSI.exe
        // https://github.com/dotnet/roslyn/blob/version-2.0.0-rc3/src/Interactive/csi/csi.rsp
        internal static readonly IEnumerable<string> DefaultNamespaces = new[]
        {
            "System",
            "System.IO",
            "System.Collections.Generic",
            "System.Console",
            "System.Diagnostics",
            "System.Dynamic",
            "System.Linq",
            "System.Linq.Expressions",
            "System.Text",
            "System.Threading.Tasks"
        };

        private static readonly CSharpParseOptions ParseOptions = new CSharpParseOptions(LanguageVersion.CSharp8, DocumentationMode.Parse, SourceCodeKind.Script);

        private readonly Lazy<CSharpCompilationOptions> _compilationOptions;
        private readonly Lazy<CSharpCommandLineArguments> _commandLineArgs;
        private readonly ScriptOptions _scriptOptions;
        private readonly IOmniSharpEnvironment _env;
        private readonly ILogger _logger;
        private readonly bool _isDesktopClr;
        private readonly bool _editorConfigEnabled;

        public ScriptProjectProvider(ScriptOptions scriptOptions, IOmniSharpEnvironment env, ILoggerFactory loggerFactory, bool isDesktopClr, bool editorConfigEnabled)
        {
            _scriptOptions = scriptOptions ?? throw new ArgumentNullException(nameof(scriptOptions));
            _env = env ?? throw new ArgumentNullException(nameof(env));

            _logger = loggerFactory.CreateLogger<ScriptProjectSystem>();
            _compilationOptions = new Lazy<CSharpCompilationOptions>(CreateCompilationOptions);
            _commandLineArgs = new Lazy<CSharpCommandLineArguments>(CreateCommandLineArguments);
            _isDesktopClr = isDesktopClr;
            _editorConfigEnabled = editorConfigEnabled;
        }

        private CSharpCommandLineArguments CreateCommandLineArguments()
        {
            if (!string.IsNullOrWhiteSpace(_scriptOptions.RspFilePath))
            {
                var rspFilePath = _scriptOptions.GetNormalizedRspFilePath(_env);
                if (rspFilePath != null)
                {
                    _logger.LogInformation($"Discovered an RSP file at '{rspFilePath}' - will use this file to discover CSX namespaces and references.");
                    return CSharpCommandLineParser.Script.Parse(new string[] { $"@{rspFilePath}" },
                        _env.TargetDirectory,
                        _isDesktopClr ? Path.GetDirectoryName(typeof(object).GetTypeInfo().Assembly.ManifestModule.FullyQualifiedName) : null);
                }
            }

            return null;
        }

        private CSharpCompilationOptions CreateCompilationOptions()
        {
            var csharpCommandLineArguments = _commandLineArgs.Value;

            // if RSP file was used, pick namespaces from there
            // otherwise use default set of namespaces
            var compilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary,
                usings: csharpCommandLineArguments != null
                    ? csharpCommandLineArguments.CompilationOptions.Usings
                    : DefaultNamespaces);

            foreach (var ns in compilationOptions.Usings)
            {
                _logger.LogDebug($"CSX global using statement: {ns}");
            }

            compilationOptions = compilationOptions
                .WithAllowUnsafe(true)
                .WithMetadataReferenceResolver(CreateMetadataReferenceResolver())
                .WithSourceReferenceResolver(ScriptSourceResolver.Default)
                .WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default)
                .WithSpecificDiagnosticOptions(!_scriptOptions.IsNugetEnabled()
                    ? CompilationOptionsHelper.GetDefaultSuppressedDiagnosticOptions()
                    : CompilationOptionsHelper.GetDefaultSuppressedDiagnosticOptions(_scriptOptions.NullableDiagnostics)); // for .NET Core 3.0 dotnet-script use extra nullable diagnostics

            var topLevelBinderFlagsProperty = typeof(CSharpCompilationOptions).GetProperty(TopLevelBinderFlagsProperty, BindingFlags.Instance | BindingFlags.NonPublic);
            var binderFlagsType = typeof(CSharpCompilationOptions).GetTypeInfo().Assembly.GetType(BinderFlagsType);

            var ignoreCorLibraryDuplicatedTypesMember = binderFlagsType?.GetField(IgnoreCorLibraryDuplicatedTypesField, BindingFlags.Static | BindingFlags.Public);
            var ignoreCorLibraryDuplicatedTypesValue = ignoreCorLibraryDuplicatedTypesMember?.GetValue(null);
            if (ignoreCorLibraryDuplicatedTypesValue != null)
            {
                topLevelBinderFlagsProperty?.SetValue(compilationOptions, ignoreCorLibraryDuplicatedTypesValue);
            }

            // in scripts, the option to supersede lower versions is ALWAYS enabled
            // see: https://github.com/dotnet/roslyn/blob/version-2.6.0-beta3/src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs#L199
            var referencesSupersedeLowerVersionsProperty = typeof(CompilationOptions).GetProperty(ReferencesSupersedeLowerVersionsProperty, BindingFlags.Instance | BindingFlags.NonPublic);
            referencesSupersedeLowerVersionsProperty?.SetValue(compilationOptions, true);

            return compilationOptions;
        }

        private CachingScriptMetadataResolver CreateMetadataReferenceResolver()
        {
            var defaultResolver = ScriptMetadataResolver.Default.WithBaseDirectory(_env.TargetDirectory);
            InjectXMLDocumentationProviderIntoRuntimeMetadataReferenceResolver(defaultResolver);

            var decoratedResolver = _scriptOptions.EnableScriptNuGetReferences
                ? new CachingScriptMetadataResolver(new NuGetMetadataReferenceResolver(defaultResolver))
                : new CachingScriptMetadataResolver(defaultResolver);

            return decoratedResolver;
        }

        public ProjectInfo CreateProject(string csxFileName, IEnumerable<MetadataReference> references, string csxFilePath, Type globalsType, IEnumerable<string> namespaces = null)
        {
            var csharpCommandLineArguments = _commandLineArgs.Value;

            // if RSP file was used, include the metadata references from RSP merged with the provided set
            // otherwise just use the provided metadata references
            if (csharpCommandLineArguments != null && csharpCommandLineArguments.MetadataReferences.Any())
            {<commit_msg>C# Scripting now uses the loadPaths and libPaths from rsp file<commit_after>﻿using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using Dotnet.Script.DependencyModel.NuGet;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.CodeAnalysis.Scripting.Hosting;
using Microsoft.Extensions.Logging;
using OmniSharp.Helpers;
using OmniSharp.Roslyn.EditorConfig;
using OmniSharp.Roslyn.Utilities;

namespace OmniSharp.Script
{
    public class ScriptProjectProvider
    {
        private const string BinderFlagsType = "Microsoft.CodeAnalysis.CSharp.BinderFlags";
        private const string TopLevelBinderFlagsProperty = "TopLevelBinderFlags";
        private const string ReferencesSupersedeLowerVersionsProperty = "ReferencesSupersedeLowerVersions_internal_protected_set";
        private const string IgnoreCorLibraryDuplicatedTypesField = "IgnoreCorLibraryDuplicatedTypes";
        private const string RuntimeMetadataReferenceResolverType = "Microsoft.CodeAnalysis.Scripting.Hosting.RuntimeMetadataReferenceResolver";
        private const string ResolverField = "_resolver";
        private const string FileReferenceProviderField = "_fileReferenceProvider";

        // aligned with CSI.exe
        // https://github.com/dotnet/roslyn/blob/version-2.0.0-rc3/src/Interactive/csi/csi.rsp
        internal static readonly IEnumerable<string> DefaultNamespaces = new[]
        {
            "System",
            "System.IO",
            "System.Collections.Generic",
            "System.Console",
            "System.Diagnostics",
            "System.Dynamic",
            "System.Linq",
            "System.Linq.Expressions",
            "System.Text",
            "System.Threading.Tasks"
        };

        private static readonly CSharpParseOptions ParseOptions = new CSharpParseOptions(LanguageVersion.CSharp8, DocumentationMode.Parse, SourceCodeKind.Script);

        private readonly Lazy<CSharpCompilationOptions> _compilationOptions;
        private readonly Lazy<CSharpCommandLineArguments> _commandLineArgs;
        private readonly ScriptOptions _scriptOptions;
        private readonly IOmniSharpEnvironment _env;
        private readonly ILogger _logger;
        private readonly bool _isDesktopClr;
        private readonly bool _editorConfigEnabled;

        public ScriptProjectProvider(ScriptOptions scriptOptions, IOmniSharpEnvironment env, ILoggerFactory loggerFactory, bool isDesktopClr, bool editorConfigEnabled)
        {
            _scriptOptions = scriptOptions ?? throw new ArgumentNullException(nameof(scriptOptions));
            _env = env ?? throw new ArgumentNullException(nameof(env));

            _logger = loggerFactory.CreateLogger<ScriptProjectSystem>();
            _compilationOptions = new Lazy<CSharpCompilationOptions>(CreateCompilationOptions);
            _commandLineArgs = new Lazy<CSharpCommandLineArguments>(CreateCommandLineArguments);
            _isDesktopClr = isDesktopClr;
            _editorConfigEnabled = editorConfigEnabled;
        }

        private CSharpCommandLineArguments CreateCommandLineArguments()
        {
            if (!string.IsNullOrWhiteSpace(_scriptOptions.RspFilePath))
            {
                var rspFilePath = _scriptOptions.GetNormalizedRspFilePath(_env);
                if (rspFilePath != null)
                {
                    _logger.LogInformation($"Discovered an RSP file at '{rspFilePath}' - will use this file to discover CSX namespaces and references.");
                    return CSharpCommandLineParser.Script.Parse(new string[] { $"@{rspFilePath}" },
                        _env.TargetDirectory,
                        _isDesktopClr ? Path.GetDirectoryName(typeof(object).GetTypeInfo().Assembly.ManifestModule.FullyQualifiedName) : null);
                }
            }

            return null;
        }

        private CSharpCompilationOptions CreateCompilationOptions()
        {
            var csharpCommandLineArguments = _commandLineArgs.Value;

            // if RSP file was used, pick namespaces from there
            // otherwise use default set of namespaces
            var compilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary,
                usings: csharpCommandLineArguments != null
                    ? csharpCommandLineArguments.CompilationOptions.Usings
                    : DefaultNamespaces);

            foreach (var ns in compilationOptions.Usings)
            {
                _logger.LogDebug($"CSX global using statement: {ns}");
            }

            var metadataReferenceResolver = CreateMetadataReferenceResolver(csharpCommandLineArguments?.ReferencePaths);
            var sourceResolver = CreateScriptSourceResolver(csharpCommandLineArguments?.SourcePaths);

            compilationOptions = compilationOptions
                .WithAllowUnsafe(true)

                .WithMetadataReferenceResolver(metadataReferenceResolver)
                .WithSourceReferenceResolver(sourceResolver)
                .WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default)
                .WithSpecificDiagnosticOptions(!_scriptOptions.IsNugetEnabled()
                    ? CompilationOptionsHelper.GetDefaultSuppressedDiagnosticOptions()
                    : CompilationOptionsHelper.GetDefaultSuppressedDiagnosticOptions(_scriptOptions.NullableDiagnostics)); // for .NET Core 3.0 dotnet-script use extra nullable diagnostics

            var topLevelBinderFlagsProperty = typeof(CSharpCompilationOptions).GetProperty(TopLevelBinderFlagsProperty, BindingFlags.Instance | BindingFlags.NonPublic);
            var binderFlagsType = typeof(CSharpCompilationOptions).GetTypeInfo().Assembly.GetType(BinderFlagsType);

            var ignoreCorLibraryDuplicatedTypesMember = binderFlagsType?.GetField(IgnoreCorLibraryDuplicatedTypesField, BindingFlags.Static | BindingFlags.Public);
            var ignoreCorLibraryDuplicatedTypesValue = ignoreCorLibraryDuplicatedTypesMember?.GetValue(null);
            if (ignoreCorLibraryDuplicatedTypesValue != null)
            {
                topLevelBinderFlagsProperty?.SetValue(compilationOptions, ignoreCorLibraryDuplicatedTypesValue);
            }

            // in scripts, the option to supersede lower versions is ALWAYS enabled
            // see: https://github.com/dotnet/roslyn/blob/version-2.6.0-beta3/src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs#L199
            var referencesSupersedeLowerVersionsProperty = typeof(CompilationOptions).GetProperty(ReferencesSupersedeLowerVersionsProperty, BindingFlags.Instance | BindingFlags.NonPublic);
            referencesSupersedeLowerVersionsProperty?.SetValue(compilationOptions, true);

            return compilationOptions;
        }

        private ScriptSourceResolver CreateScriptSourceResolver(IEnumerable<string> searchPaths)
        {
            var defaultResolver = ScriptSourceResolver.Default;

            if (searchPaths == null)
            {
                return defaultResolver;
            }

            defaultResolver = defaultResolver.WithSearchPaths(searchPaths);

            foreach (string path in searchPaths)
            {
                _logger.LogInformation($"CSX source path: {path}.");
            }

            return defaultResolver;
        }

        private CachingScriptMetadataResolver CreateMetadataReferenceResolver(IEnumerable<string> searchPaths)
        {
            var defaultResolver = ScriptMetadataResolver.Default.WithBaseDirectory(_env.TargetDirectory);

            if (searchPaths != null)
            {
                defaultResolver = defaultResolver.WithSearchPaths(searchPaths);

                foreach (string path in searchPaths)
                {
                    _logger.LogInformation($"CSX reference path: {path}.");
                }
            }

            InjectXMLDocumentationProviderIntoRuntimeMetadataReferenceResolver(defaultResolver);

            var decoratedResolver = _scriptOptions.EnableScriptNuGetReferences
                ? new CachingScriptMetadataResolver(new NuGetMetadataReferenceResolver(defaultResolver))
                : new CachingScriptMetadataResolver(defaultResolver);

            return decoratedResolver;
        }

        public ProjectInfo CreateProject(string csxFileName, IEnumerable<MetadataReference> references, string csxFilePath, Type globalsType, IEnumerable<string> namespaces = null)
        {
            var csharpCommandLineArguments = _commandLineArgs.Value;

            // if RSP file was used, include the metadata references from RSP merged with the provided set
            // otherwise just use the provided metadata references
            if (csharpCommandLineArguments != null && csharpCommandLineArguments.MetadataReferences.Any())
            {
{'fraction_non_alphanumeric': 0.06653165763297397, 'fraction_numerical': 0.0016677586514980047, 'mean_word_length': 3.012906309751434, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6023419', 'n_tokens_mistral': 3981, 'n_tokens_neox': 3795, 'n_words': 776}