<commit_before>// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Dfm
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Concurrent;
    using System.IO;
    using System.Linq;
    using System.Text.RegularExpressions;

    public class TagNameBlockPathQueryOption : DfmFencesBlockPathQueryOption
    {
        // C family code snippet comment block: // <[/]snippetname>
        private static readonly Regex CFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\/{2}\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex CFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\/{2}\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Basic family code snippet comment block: ' <[/]snippetname>
        private static readonly Regex BasicFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\'\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex BasicFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\'\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Markup language family code snippet block: <!-- <[/]snippetname> -->
        private static readonly Regex MarkupLanguageFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\<\!\-{2}\s*\<\s*(?<name>[\w\.]+)\s*\>\s*\-{2}\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex MarkupLanguageFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\<\!\-{2}\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*\-{2}\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Sql family code snippet block: -- <[/]snippetname>
        private static readonly Regex SqlFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\-{2}\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex SqlFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\-{2}\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Script family snippet comment block: # <[/]snippetname>
        private static readonly Regex ScriptFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*#\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex ScriptFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*#\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Lisp code snippet comment block: rem <[/]snippetname>
        private static readonly Regex BatchFileCodeSnippetRegionStartLineRegex = new Regex(@"^\s*rem\s+\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex BatchFileCodeSnippetRegionEndLineRegex = new Regex(@"^\s*rem\s+\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // C# code snippet region block: start -> #region snippetname, end -> #endregion
        private static readonly Regex CSharpCodeSnippetRegionStartLineRegex = new Regex(@"^\s*#\s*region(?:\s+(?<name>.+?))?\s*$", RegexOptions.Compiled);
        private static readonly Regex CSharpCodeSnippetRegionEndLineRegex = new Regex(@"^\s*#\s*endregion(?:\s.*)?$", RegexOptions.Compiled);

        // Erlang code snippet comment block: % <[/]snippetname>
        private static readonly Regex ErlangCodeSnippetRegionStartLineRegex = new Regex(@"^\s*%\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex ErlangCodeSnippetRegionEndLineRegex = new Regex(@"^\s*%\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Lisp code snippet comment block: ; <[/]snippetname>
        private static readonly Regex LispCodeSnippetRegionStartLineRegex = new Regex(@"^\s*;\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex LispCodeSnippetRegionEndLineRegex = new Regex(@"^\s*;\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // VB code snippet Region block: start -> # Region "snippetname", end -> # End Region
        private static readonly Regex VBCodeSnippetRegionRegionStartLineRegex = new Regex(@"^\s*#\s*Region(?:\s+(?<name>.+?))?\s*$", RegexOptions.Compiled);
        private static readonly Regex VBCodeSnippetRegionRegionEndLineRegex = new Regex(@"^\s*#\s*End\s+Region(?:\s.*)?$", RegexOptions.Compiled);

        // Language names and aliases fllow http://highlightjs.readthedocs.org/en/latest/css-classes-reference.html#language-names-and-aliases
        // Language file extensions follow https://github.com/github/linguist/blob/master/lib/linguist/languages.yml
        // Currently only supports parts of the language names, aliases and extensions
        // Later we can move the repository's supported/custom language names, aliases, extensions and corresponding comments regexes to docfx build configuration
        private static readonly IReadOnlyDictionary<string, List<ICodeSnippetExtractor>> CodeLanguageExtractors = GetCodeLanguageExtractors();

        public string TagName { get; set; }

        private DfmTagNameResolveResult _resolveResult;

        private readonly ConcurrentDictionary<string, Lazy<ConcurrentDictionary<string, List<DfmTagNameResolveResult>>>> _dfmTagNameLineRangeCache =
            new ConcurrentDictionary<string, Lazy<ConcurrentDictionary<string, List<DfmTagNameResolveResult>>>>(StringComparer.OrdinalIgnoreCase);

        private static Dictionary<string, List<ICodeSnippetExtractor>> GetCodeLanguageExtractors()
        {
            return new CodeLanguageExtractorsBuilder()
                .AddAlias("actionscript", ".as")
                .AddAlias("arduino", ".ino")
                .AddAlias("assembly", "nasm", ".asm")
                .AddAlias("batchfile", ".bat", ".cmd")
                .AddAlias("cpp", "c", "c++", "objective-c", "obj-c", "objc", "objectivec", ".c", ".cpp", ".h", ".hpp", ".cc")
                .AddAlias("csharp", "cs", ".cs")
                .AddAlias("cuda", ".cu", ".cuh")
                .AddAlias("d", "dlang", ".d")
                .AddAlias("erlang", ".erl")
                .AddAlias("fsharp", "fs", ".fs", ".fsi", ".fsx")
                .AddAlias("go", "golang", ".go")
                .AddAlias("haskell", ".hs")
                .AddAlias("html", ".html", ".cshtml", "cshtml", ".vbhtml", "vbhtml", ".jsp", ".asp", ".aspx", "ascx")
                .AddAlias("java", ".java")
                .AddAlias("javascript", "js", "node", ".js")
                .AddAlias("lisp", ".lisp", ".lsp")
                .AddAlias("lua", ".lua")
                .AddAlias("matlab", ".matlab")
                .AddAlias("pascal", ".pas")
                .AddAlias("perl", ".pl")
                .AddAlias("php", ".php")
                .AddAlias("powershell", "posh", ".ps1")
                .AddAlias("processing", ".pde")
                .AddAlias("python", ".py")<commit_msg>add language support for aspx-cs/vb. (#1908)<commit_after>// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Dfm
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Concurrent;
    using System.IO;
    using System.Linq;
    using System.Text.RegularExpressions;

    public class TagNameBlockPathQueryOption : DfmFencesBlockPathQueryOption
    {
        // C family code snippet comment block: // <[/]snippetname>
        private static readonly Regex CFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\/{2}\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex CFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\/{2}\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Basic family code snippet comment block: ' <[/]snippetname>
        private static readonly Regex BasicFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\'\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex BasicFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\'\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Markup language family code snippet block: <!-- <[/]snippetname> -->
        private static readonly Regex MarkupLanguageFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\<\!\-{2}\s*\<\s*(?<name>[\w\.]+)\s*\>\s*\-{2}\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex MarkupLanguageFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\<\!\-{2}\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*\-{2}\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Sql family code snippet block: -- <[/]snippetname>
        private static readonly Regex SqlFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*\-{2}\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex SqlFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*\-{2}\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Script family snippet comment block: # <[/]snippetname>
        private static readonly Regex ScriptFamilyCodeSnippetCommentStartLineRegex = new Regex(@"^\s*#\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex ScriptFamilyCodeSnippetCommentEndLineRegex = new Regex(@"^\s*#\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Lisp code snippet comment block: rem <[/]snippetname>
        private static readonly Regex BatchFileCodeSnippetRegionStartLineRegex = new Regex(@"^\s*rem\s+\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex BatchFileCodeSnippetRegionEndLineRegex = new Regex(@"^\s*rem\s+\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // C# code snippet region block: start -> #region snippetname, end -> #endregion
        private static readonly Regex CSharpCodeSnippetRegionStartLineRegex = new Regex(@"^\s*#\s*region(?:\s+(?<name>.+?))?\s*$", RegexOptions.Compiled);
        private static readonly Regex CSharpCodeSnippetRegionEndLineRegex = new Regex(@"^\s*#\s*endregion(?:\s.*)?$", RegexOptions.Compiled);

        // Erlang code snippet comment block: % <[/]snippetname>
        private static readonly Regex ErlangCodeSnippetRegionStartLineRegex = new Regex(@"^\s*%\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex ErlangCodeSnippetRegionEndLineRegex = new Regex(@"^\s*%\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // Lisp code snippet comment block: ; <[/]snippetname>
        private static readonly Regex LispCodeSnippetRegionStartLineRegex = new Regex(@"^\s*;\s*\<\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
        private static readonly Regex LispCodeSnippetRegionEndLineRegex = new Regex(@"^\s*;\s*\<\s*\/\s*(?<name>[\w\.]+)\s*\>\s*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);

        // VB code snippet Region block: start -> # Region "snippetname", end -> # End Region
        private static readonly Regex VBCodeSnippetRegionRegionStartLineRegex = new Regex(@"^\s*#\s*Region(?:\s+(?<name>.+?))?\s*$", RegexOptions.Compiled);
        private static readonly Regex VBCodeSnippetRegionRegionEndLineRegex = new Regex(@"^\s*#\s*End\s+Region(?:\s.*)?$", RegexOptions.Compiled);

        // Language names and aliases fllow http://highlightjs.readthedocs.org/en/latest/css-classes-reference.html#language-names-and-aliases
        // Language file extensions follow https://github.com/github/linguist/blob/master/lib/linguist/languages.yml
        // Currently only supports parts of the language names, aliases and extensions
        // Later we can move the repository's supported/custom language names, aliases, extensions and corresponding comments regexes to docfx build configuration
        private static readonly IReadOnlyDictionary<string, List<ICodeSnippetExtractor>> CodeLanguageExtractors = GetCodeLanguageExtractors();

        public string TagName { get; set; }

        private DfmTagNameResolveResult _resolveResult;

        private readonly ConcurrentDictionary<string, Lazy<ConcurrentDictionary<string, List<DfmTagNameResolveResult>>>> _dfmTagNameLineRangeCache =
            new ConcurrentDictionary<string, Lazy<ConcurrentDictionary<string, List<DfmTagNameResolveResult>>>>(StringComparer.OrdinalIgnoreCase);

        private static Dictionary<string, List<ICodeSnippetExtractor>> GetCodeLanguageExtractors()
        {
            return new CodeLanguageExtractorsBuilder()
                .AddAlias("actionscript", ".as")
                .AddAlias("arduino", ".ino")
                .AddAlias("assembly", "nasm", ".asm")
                .AddAlias("batchfile", ".bat", ".cmd")
                .AddAlias("cpp", "c", "c++", "objective-c", "obj-c", "objc", "objectivec", ".c", ".cpp", ".h", ".hpp", ".cc")
                .AddAlias("csharp", "cs", ".cs")
                .AddAlias("cuda", ".cu", ".cuh")
                .AddAlias("d", "dlang", ".d")
                .AddAlias("erlang", ".erl")
                .AddAlias("fsharp", "fs", ".fs", ".fsi", ".fsx")
                .AddAlias("go", "golang", ".go")
                .AddAlias("haskell", ".hs")
                .AddAlias("html", ".html", ".cshtml", "cshtml", ".vbhtml", "vbhtml", ".jsp", "aspx-cs", "aspx-csharp", "aspx-vb" ,".asp", ".aspx", ".ascx")
                .AddAlias("java", ".java")
                .AddAlias("javascript", "js", "node", ".js")
                .AddAlias("lisp", ".lisp", ".lsp")
                .AddAlias("lua", ".lua")
                .AddAlias("matlab", ".matlab")
                .AddAlias("pascal", ".pas")
                .AddAlias("perl", ".pl")
                .AddAlias("php", ".php")
                .AddAlias("powershell", "posh", ".ps1")
                .AddAlias("processing", ".pde")
                .AddAlias("python", ".py")
{'fraction_non_alphanumeric': 0.18499562260084854, 'fraction_numerical': 0.0014815812512627112, 'mean_word_length': 3.316860465116279, 'pattern_counts': {'":': 0, '<': 117, '<?xml version=': 0, '>': 125, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114526', 'n_tokens_mistral': 4712, 'n_tokens_neox': 4556, 'n_words': 893}