<commit_before>// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
#if NET46
using System.Runtime.Remoting;
using System.Runtime.Remoting.Messaging;
#else
using System.Threading;
#endif
using Microsoft.AspNetCore.Razor.Language.CodeGeneration;
using Microsoft.AspNetCore.Razor.Language.Intermediate;
using Xunit;
using Xunit.Sdk;
using Microsoft.AspNetCore.Razor.Language.Legacy;

namespace Microsoft.AspNetCore.Razor.Language.IntegrationTests
{
    [IntializeTestFile]
    public abstract class IntegrationTestBase
    {
#if !NET46
        private static readonly AsyncLocal<string> _fileName = new AsyncLocal<string>();
#endif

        protected IntegrationTestBase()
        {
            TestProjectRoot = TestProject.GetProjectDirectory(GetType());
        }

#if GENERATE_BASELINES
        protected bool GenerateBaselines { get; set; } = true;
#else
        protected bool GenerateBaselines { get; set; } = false;
#endif

        protected string TestProjectRoot { get; }

        // Used by the test framework to set the 'base' name for test files.
        public static string FileName
        {
#if NET46
            get
            {
                var handle = (ObjectHandle)CallContext.LogicalGetData("IntegrationTestBase_FileName");
                return (string)handle.Unwrap();
            }
            set
            {
                CallContext.LogicalSetData("IntegrationTestBase_FileName", new ObjectHandle(value));
            }
#elif NETCOREAPP2_2
            get { return _fileName.Value; }
            set { _fileName.Value = value; }
#endif
        }

        protected virtual RazorProjectEngine CreateProjectEngine() => CreateProjectEngine(configure: null);

        protected virtual RazorProjectEngine CreateProjectEngine(Action<RazorProjectEngineBuilder> configure)
        {
            if (FileName == null)
            {
                var message = $"{nameof(CreateProjectEngine)} should only be called from an integration test, ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var assembly = GetType().GetTypeInfo().Assembly;
            var projectEngine = RazorProjectEngine.Create(RazorConfiguration.Default, IntegrationTestFileSystem.Default, b =>
            {
                configure?.Invoke(b);

                var existingImportFeature = b.Features.OfType<IImportProjectFeature>().Single();
                b.SetImportFeature(new IntegrationTestImportFeature(assembly, existingImportFeature));
            });
            var testProjectEngine = new IntegrationTestProjectEngine(projectEngine);

            return testProjectEngine;
        }

        protected virtual RazorProjectItem CreateProjectItem()
        {
            if (FileName == null)
            {
                var message = $"{nameof(CreateProjectItem)} should only be called from an integration test, ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var suffixIndex = FileName.LastIndexOf("_");
            var normalizedFileName = suffixIndex == -1 ? FileName : FileName.Substring(0, suffixIndex);
            var sourceFileName = Path.ChangeExtension(normalizedFileName, ".cshtml");
            var testFile = TestFile.Create(sourceFileName, GetType().GetTypeInfo().Assembly);
            if (!testFile.Exists())
            {
                throw new XunitException($"The resource {sourceFileName} was not found.");
            }
            var fileContent = testFile.ReadAllText();
            var normalizedContent = NormalizeNewLines(fileContent);

            var projectItem = new TestRazorProjectItem(sourceFileName)
            {
                Content = normalizedContent,
            };

            return projectItem;
        }

        protected void AssertDocumentNodeMatchesBaseline(DocumentIntermediateNode document)
        {
            if (FileName == null)
            {
                var message = $"{nameof(AssertDocumentNodeMatchesBaseline)} should only be called from an integration test ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var baselineFileName = Path.ChangeExtension(FileName, ".ir.txt");

            if (GenerateBaselines)
            {
                var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName);
                File.WriteAllText(baselineFullPath, IntermediateNodeSerializer.Serialize(document));
                return;
            }

            var irFile = TestFile.Create(baselineFileName, GetType().GetTypeInfo().Assembly);
            if (!irFile.Exists())
            {
                throw new XunitException($"The resource {baselineFileName} was not found.");
            }

            var baseline = irFile.ReadAllText().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
            IntermediateNodeVerifier.Verify(document, baseline);
        }

        protected void AssertCSharpDocumentMatchesBaseline(RazorCSharpDocument cSharpDocument)
        {
            if (FileName == null)
            {
                var message = $"{nameof(AssertCSharpDocumentMatchesBaseline)} should only be called from an integration test ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var baselineFileName = Path.ChangeExtension(FileName, ".codegen.cs");
            var baselineDiagnosticsFileName = Path.ChangeExtension(FileName, ".diagnostics.txt");

            if (GenerateBaselines)
            {
                var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName);
                File.WriteAllText(baselineFullPath, cSharpDocument.GeneratedCode);

                var baselineDiagnosticsFullPath = Path.Combine(TestProjectRoot, baselineDiagnosticsFileName);
                var lines = cSharpDocument.Diagnostics.Select(RazorDiagnosticSerializer.Serialize).ToArray();
                if (lines.Any())
                {
                    File.WriteAllLines(baselineDiagnosticsFullPath, lines);
                }
                else if (File.Exists(baselineDiagnosticsFullPath))
                {
                    File.Delete(baselineDiagnosticsFullPath);
                }

                return;
            }

            var codegenFile = TestFile.Create(baselineFileName, GetType().GetTypeInfo().Assembly);
            if (!codegenFile.Exists())
            {
                throw new XunitException($"The resource {baselineFileName} was not found.");
            }

            var baseline = codegenFile.ReadAllText();

            // Normalize newlines to match those in the baseline.
            var actual = cSharpDocument.GeneratedCode.Replace("\r", "").Replace("\n", "\r\n");
            Assert.Equal(baseline, actual);

            var baselineDiagnostics = string.Empty;
            var diagnosticsFile = TestFile.Create(baselineDiagnosticsFileName, GetType().GetTypeInfo().Assembly);
            if (diagnosticsFile.Exists())
            {
                baselineDiagnostics = diagnosticsFile.ReadAllText();
            }

            var actualDiagnostics = string.Concat(cSharpDocument.Diagnostics.Select(d => RazorDiagnosticSerializer.Serialize(d) + "\r\n"));
            Assert.Equal(baselineDiagnostics, actualDiagnostics);
        }

        protected void AssertSourceMappingsMatchBaseline(RazorCodeDocument codeDocument)
        {
            if (FileName == null)
            {
                var message = $"{nameof(AssertSourceMappingsMatchBaseline)} should only be called from an integration test ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var csharpDocument = codeDocument.GetCSharpDocument();
            Assert.NotNull(csharpDocument);

            var baselineFileName = Path.ChangeExtension(FileName, ".mappings.txt");
            var serializedMappings = SourceMappingsSerializer.Serialize(csharpDocument, codeDocument.Source);

            if (GenerateBaselines)
            {
                var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName);
                File.WriteAllText(baselineFullPath, serializedMappings);
                return;
            }

            var testFile = TestFile.Create(baselineFileName, GetType().GetTypeInfo().Assembly);
            if (!testFile.Exists())
            {
                throw new XunitException($"The resource {baselineFileName} was not found.");
            }

            var baseline = testFile.ReadAllText();

            // Normalize newlines to match those in the baseline.
            var actualBaseline = serializedMappings.Replace("\r", "").Replace("\n", "\r\n");

            Assert.Equal(baseline, actualBaseline);

            var syntaxTree = codeDocument.GetSyntaxTree();
            var visitor = new CodeSpanVisitor();
            visitor.VisitBlock(syntaxTree.Root);

            var charBuffer = new char[codeDocument.Source.Length];
            codeDocument.Source.CopyTo(0, charBuffer, 0, codeDocument.Source.Length);
            var sourceContent = new string(charBuffer);

            var spans = visitor.CodeSpans;
            for (var i= 0; i < spans.Count; i++)
            {
                var span = spans[i];
                if (span.Start.FilePath == null || span.Start.FilePath != codeDocument.Source.FilePath)
                {
                    // Not in the main file, skip.
                    continue;
                }

                var location = new SourceSpan(span.Start, span.Length);
                var expectedSpan = sourceContent.Substring(span.Start.AbsoluteIndex, span.Length);

                // See #
                if (string.IsNullOrWhiteSpace(expectedSpan))
                {
                    // For now we don't verify whitespace inside of a directive. We know that directives cheat
                    // with how they bound whitespace/C#/markup to make completion work.
                    if (span.Parent is Block block && block.Type == BlockKindInternal.Directive)
                    {
                        continue;
                    }
                }

                // See #
                if (string.Equals("@", expectedSpan) && span.Kind == SpanKindInternal.Code)
                {
                    // For now we don't verify an escaped transition. In some cases one of the @ tokens in @@foo
                    // will be mapped as C# but will not be present in the output buffer because it's not actually C#.
                    continue;
                }

                var found = false;
                for (var j = 0; j < csharpDocument.SourceMappings.Count; j++)<commit_msg>Add Issue numbers<commit_after>// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
#if NET46
using System.Runtime.Remoting;
using System.Runtime.Remoting.Messaging;
#else
using System.Threading;
#endif
using Microsoft.AspNetCore.Razor.Language.CodeGeneration;
using Microsoft.AspNetCore.Razor.Language.Intermediate;
using Xunit;
using Xunit.Sdk;
using Microsoft.AspNetCore.Razor.Language.Legacy;

namespace Microsoft.AspNetCore.Razor.Language.IntegrationTests
{
    [IntializeTestFile]
    public abstract class IntegrationTestBase
    {
#if !NET46
        private static readonly AsyncLocal<string> _fileName = new AsyncLocal<string>();
#endif

        protected IntegrationTestBase()
        {
            TestProjectRoot = TestProject.GetProjectDirectory(GetType());
        }

#if GENERATE_BASELINES
        protected bool GenerateBaselines { get; set; } = true;
#else
        protected bool GenerateBaselines { get; set; } = false;
#endif

        protected string TestProjectRoot { get; }

        // Used by the test framework to set the 'base' name for test files.
        public static string FileName
        {
#if NET46
            get
            {
                var handle = (ObjectHandle)CallContext.LogicalGetData("IntegrationTestBase_FileName");
                return (string)handle.Unwrap();
            }
            set
            {
                CallContext.LogicalSetData("IntegrationTestBase_FileName", new ObjectHandle(value));
            }
#elif NETCOREAPP2_2
            get { return _fileName.Value; }
            set { _fileName.Value = value; }
#endif
        }

        protected virtual RazorProjectEngine CreateProjectEngine() => CreateProjectEngine(configure: null);

        protected virtual RazorProjectEngine CreateProjectEngine(Action<RazorProjectEngineBuilder> configure)
        {
            if (FileName == null)
            {
                var message = $"{nameof(CreateProjectEngine)} should only be called from an integration test, ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var assembly = GetType().GetTypeInfo().Assembly;
            var projectEngine = RazorProjectEngine.Create(RazorConfiguration.Default, IntegrationTestFileSystem.Default, b =>
            {
                configure?.Invoke(b);

                var existingImportFeature = b.Features.OfType<IImportProjectFeature>().Single();
                b.SetImportFeature(new IntegrationTestImportFeature(assembly, existingImportFeature));
            });
            var testProjectEngine = new IntegrationTestProjectEngine(projectEngine);

            return testProjectEngine;
        }

        protected virtual RazorProjectItem CreateProjectItem()
        {
            if (FileName == null)
            {
                var message = $"{nameof(CreateProjectItem)} should only be called from an integration test, ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var suffixIndex = FileName.LastIndexOf("_");
            var normalizedFileName = suffixIndex == -1 ? FileName : FileName.Substring(0, suffixIndex);
            var sourceFileName = Path.ChangeExtension(normalizedFileName, ".cshtml");
            var testFile = TestFile.Create(sourceFileName, GetType().GetTypeInfo().Assembly);
            if (!testFile.Exists())
            {
                throw new XunitException($"The resource {sourceFileName} was not found.");
            }
            var fileContent = testFile.ReadAllText();
            var normalizedContent = NormalizeNewLines(fileContent);

            var projectItem = new TestRazorProjectItem(sourceFileName)
            {
                Content = normalizedContent,
            };

            return projectItem;
        }

        protected void AssertDocumentNodeMatchesBaseline(DocumentIntermediateNode document)
        {
            if (FileName == null)
            {
                var message = $"{nameof(AssertDocumentNodeMatchesBaseline)} should only be called from an integration test ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var baselineFileName = Path.ChangeExtension(FileName, ".ir.txt");

            if (GenerateBaselines)
            {
                var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName);
                File.WriteAllText(baselineFullPath, IntermediateNodeSerializer.Serialize(document));
                return;
            }

            var irFile = TestFile.Create(baselineFileName, GetType().GetTypeInfo().Assembly);
            if (!irFile.Exists())
            {
                throw new XunitException($"The resource {baselineFileName} was not found.");
            }

            var baseline = irFile.ReadAllText().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
            IntermediateNodeVerifier.Verify(document, baseline);
        }

        protected void AssertCSharpDocumentMatchesBaseline(RazorCSharpDocument cSharpDocument)
        {
            if (FileName == null)
            {
                var message = $"{nameof(AssertCSharpDocumentMatchesBaseline)} should only be called from an integration test ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var baselineFileName = Path.ChangeExtension(FileName, ".codegen.cs");
            var baselineDiagnosticsFileName = Path.ChangeExtension(FileName, ".diagnostics.txt");

            if (GenerateBaselines)
            {
                var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName);
                File.WriteAllText(baselineFullPath, cSharpDocument.GeneratedCode);

                var baselineDiagnosticsFullPath = Path.Combine(TestProjectRoot, baselineDiagnosticsFileName);
                var lines = cSharpDocument.Diagnostics.Select(RazorDiagnosticSerializer.Serialize).ToArray();
                if (lines.Any())
                {
                    File.WriteAllLines(baselineDiagnosticsFullPath, lines);
                }
                else if (File.Exists(baselineDiagnosticsFullPath))
                {
                    File.Delete(baselineDiagnosticsFullPath);
                }

                return;
            }

            var codegenFile = TestFile.Create(baselineFileName, GetType().GetTypeInfo().Assembly);
            if (!codegenFile.Exists())
            {
                throw new XunitException($"The resource {baselineFileName} was not found.");
            }

            var baseline = codegenFile.ReadAllText();

            // Normalize newlines to match those in the baseline.
            var actual = cSharpDocument.GeneratedCode.Replace("\r", "").Replace("\n", "\r\n");
            Assert.Equal(baseline, actual);

            var baselineDiagnostics = string.Empty;
            var diagnosticsFile = TestFile.Create(baselineDiagnosticsFileName, GetType().GetTypeInfo().Assembly);
            if (diagnosticsFile.Exists())
            {
                baselineDiagnostics = diagnosticsFile.ReadAllText();
            }

            var actualDiagnostics = string.Concat(cSharpDocument.Diagnostics.Select(d => RazorDiagnosticSerializer.Serialize(d) + "\r\n"));
            Assert.Equal(baselineDiagnostics, actualDiagnostics);
        }

        protected void AssertSourceMappingsMatchBaseline(RazorCodeDocument codeDocument)
        {
            if (FileName == null)
            {
                var message = $"{nameof(AssertSourceMappingsMatchBaseline)} should only be called from an integration test ({nameof(FileName)} is null).";
                throw new InvalidOperationException(message);
            }

            var csharpDocument = codeDocument.GetCSharpDocument();
            Assert.NotNull(csharpDocument);

            var baselineFileName = Path.ChangeExtension(FileName, ".mappings.txt");
            var serializedMappings = SourceMappingsSerializer.Serialize(csharpDocument, codeDocument.Source);

            if (GenerateBaselines)
            {
                var baselineFullPath = Path.Combine(TestProjectRoot, baselineFileName);
                File.WriteAllText(baselineFullPath, serializedMappings);
                return;
            }

            var testFile = TestFile.Create(baselineFileName, GetType().GetTypeInfo().Assembly);
            if (!testFile.Exists())
            {
                throw new XunitException($"The resource {baselineFileName} was not found.");
            }

            var baseline = testFile.ReadAllText();

            // Normalize newlines to match those in the baseline.
            var actualBaseline = serializedMappings.Replace("\r", "").Replace("\n", "\r\n");

            Assert.Equal(baseline, actualBaseline);

            var syntaxTree = codeDocument.GetSyntaxTree();
            var visitor = new CodeSpanVisitor();
            visitor.VisitBlock(syntaxTree.Root);

            var charBuffer = new char[codeDocument.Source.Length];
            codeDocument.Source.CopyTo(0, charBuffer, 0, codeDocument.Source.Length);
            var sourceContent = new string(charBuffer);

            var spans = visitor.CodeSpans;
            for (var i= 0; i < spans.Count; i++)
            {
                var span = spans[i];
                if (span.Start.FilePath == null || span.Start.FilePath != codeDocument.Source.FilePath)
                {
                    // Not in the main file, skip.
                    continue;
                }

                var location = new SourceSpan(span.Start, span.Length);
                var expectedSpan = sourceContent.Substring(span.Start.AbsoluteIndex, span.Length);

                // See #
                if (string.IsNullOrWhiteSpace(expectedSpan))
                {
                    // For now we don't verify whitespace inside of a directive. We know that directives cheat
                    // with how they bound whitespace/C#/markup to make completion work.
                    if (span.Parent is Block block && block.Type == BlockKindInternal.Directive)
                    {
                        continue;
                    }
                }

                // See #2594
                if (string.Equals("@", expectedSpan) && span.Kind == SpanKindInternal.Code)
                {
                    // For now we don't verify an escaped transition. In some cases one of the @ tokens in @@foo
                    // will be mapped as C# but will not be present in the output buffer because it's not actually C#.
                    continue;
                }

                var found = false;
                for (var j = 0; j < csharpDocument.SourceMappings.Count; j++)
{'fraction_non_alphanumeric': 0.08873226710038855, 'fraction_numerical': 0.001626457034426674, 'mean_word_length': 2.078153247114449, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1923863', 'n_tokens_mistral': 5319, 'n_tokens_neox': 5092, 'n_words': 1278}