<commit_before>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using Microsoft.CodeAnalysis.LanguageServices;
using Microsoft.CodeAnalysis.Shared.Extensions;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.RegularExpressions
{
    /// <summary>
    /// Helper class to detect regex pattern tokens in a document efficiently.
    /// </summary>
    internal class RegexPatternDetector
    {
        private const string _patternName = "pattern";

        private readonly SemanticModel _semanticModel;
        private readonly ISyntaxFactsService _syntaxFacts;
        private readonly ISemanticFactsService _semanticFacts;
        private readonly INamedTypeSymbol _regexType;
        private readonly HashSet<string> _methodNamesOfInterest;

        public RegexPatternDetector(
            SemanticModel semanticModel, 
            ISyntaxFactsService syntaxFacts,
            ISemanticFactsService semanticFacts,
            INamedTypeSymbol regexType, 
            HashSet<string> methodNamesOfInterest)
        {
            _semanticModel = semanticModel;
            _syntaxFacts = syntaxFacts;
            _semanticFacts = semanticFacts;
            _regexType = regexType;
            _methodNamesOfInterest = methodNamesOfInterest;
        }

        public static RegexPatternDetector TryCreate(
            SemanticModel semanticModel, 
            ISyntaxFactsService syntaxFacts, 
            ISemanticFactsService semanticFacts)
        {
            var regexType = semanticModel.Compilation.GetTypeByMetadataName(typeof(Regex).FullName);
            if (regexType == null)
            {
                return null;
            }

            var methodNamesOfInterest = GetMethodNamesOfInterest(regexType, syntaxFacts);
            return new RegexPatternDetector(
                semanticModel, syntaxFacts, semanticFacts,
                regexType, methodNamesOfInterest);
        }

        public static bool IsDefinitelyNotPattern(SyntaxToken token, ISyntaxFactsService syntaxFacts)
        {
            // We only support string literals passed in arguments to something.
            // In the future we could support any string literal, as long as it has
            // some marker (like a comment on it) stating it's a regex.
            if (!syntaxFacts.IsStringLiteral(token))
            {
                return true;
            }

            if (!IsMethodOrConstructorArgument(token, syntaxFacts) && 
                !HasRegexLanguageComment(token, syntaxFacts, out _))
            {
                return true;
            }

            return false;
        }

        private static bool HasRegexLanguageComment(
            SyntaxToken token, ISyntaxFactsService syntaxFacts, out RegexOptions options)
        {<commit_msg>Allow users to mark which strings are regexes<commit_after>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using Microsoft.CodeAnalysis.LanguageServices;
using Microsoft.CodeAnalysis.Shared.Extensions;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.RegularExpressions
{
    /// <summary>
    /// Helper class to detect regex pattern tokens in a document efficiently.
    /// </summary>
    internal class RegexPatternDetector
    {
        private const string _patternName = "pattern";

        private readonly SemanticModel _semanticModel;
        private readonly ISyntaxFactsService _syntaxFacts;
        private readonly ISemanticFactsService _semanticFacts;
        private readonly INamedTypeSymbol _regexType;
        private readonly HashSet<string> _methodNamesOfInterest;

        public RegexPatternDetector(
            SemanticModel semanticModel, 
            ISyntaxFactsService syntaxFacts,
            ISemanticFactsService semanticFacts,
            INamedTypeSymbol regexType, 
            HashSet<string> methodNamesOfInterest)
        {
            _semanticModel = semanticModel;
            _syntaxFacts = syntaxFacts;
            _semanticFacts = semanticFacts;
            _regexType = regexType;
            _methodNamesOfInterest = methodNamesOfInterest;
        }

        public static RegexPatternDetector TryCreate(
            SemanticModel semanticModel, 
            ISyntaxFactsService syntaxFacts, 
            ISemanticFactsService semanticFacts)
        {
            var regexType = semanticModel.Compilation.GetTypeByMetadataName(typeof(Regex).FullName);
            if (regexType == null)
            {
                return null;
            }

            var methodNamesOfInterest = GetMethodNamesOfInterest(regexType, syntaxFacts);
            return new RegexPatternDetector(
                semanticModel, syntaxFacts, semanticFacts,
                regexType, methodNamesOfInterest);
        }

        public static bool IsDefinitelyNotPattern(SyntaxToken token, ISyntaxFactsService syntaxFacts)
        {
            // We only support string literals passed in arguments to something.
            // In the future we could support any string literal, as long as it has
            // some marker (like a comment on it) stating it's a regex.
            if (!syntaxFacts.IsStringLiteral(token))
            {
                return true;
            }

            if (!IsMethodOrConstructorArgument(token, syntaxFacts) && 
                !HasRegexLanguageComment(token, syntaxFacts, out _))
            {
                return true;
            }

            return false;
        }

        private static bool HasRegexLanguageComment(
            SyntaxToken token, ISyntaxFactsService syntaxFacts, out RegexOptions options)
        {
            if (HasRegexLanguageComment(token.GetPreviousToken().TrailingTrivia, syntaxFacts, out options))
            {
                return true;
            }

{'fraction_non_alphanumeric': 0.055873925501432664, 'fraction_numerical': 0.0006367398917542184, 'mean_word_length': 2.320824524312896, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '419945', 'n_tokens_mistral': 1564, 'n_tokens_neox': 1488, 'n_words': 420}