<commit_before>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Elfie.Model;
using Microsoft.CodeAnalysis.Packaging;
using Microsoft.CodeAnalysis.Shared.Options;
using Microsoft.CodeAnalysis.Shared.Utilities;
using Microsoft.Internal.VisualStudio.Shell.Interop;
using Roslyn.Utilities;
using static System.FormattableString;

namespace Microsoft.VisualStudio.LanguageServices.SymbolSearch
{
    /// <summary>
    /// A service which enables searching for packages matching certain criteria.
    /// It works against an <see cref="Microsoft.CodeAnalysis.Elfie"/> database to find results.
    /// 
    /// This implementation also spawns a task which will attempt to keep that database up to
    /// date by downloading patches on a daily basis.
    /// </summary>
    internal partial class SymbolSearchService
    {
        // Internal for testing purposes.
        internal const string ContentAttributeName = "content";
        internal const string ChecksumAttributeName = "checksum";
        internal const string UpToDateAttributeName = "upToDate";
        internal const string TooOldAttributeName = "tooOld";
        internal const string NugetOrgSource = "nuget.org";

        private const string HostId = "RoslynNuGetSearch";
        private const string MicrosoftAssemblyReferencesName = "MicrosoftAssemblyReferences";
        private static readonly LinkedList<string> s_log = new LinkedList<string>();

        /// <summary>
        /// Cancellation support for the task we use to keep the local database up to date.
        /// When VS shuts down it will dispose us.  We'll cancel the task at that point.
        /// </summary>
        private readonly CancellationTokenSource _cancellationTokenSource;
        private readonly CancellationToken _cancellationToken;

        private readonly ConcurrentDictionary<string, object> _sourceToUpdateSentinel =
            new ConcurrentDictionary<string, object>();

        // Interfaces that abstract out the external functionality we need.  Used so we can easily
        // mock behavior during tests.
        private readonly IPackageInstallerService _installerService;
        private readonly IDelayService _delayService;
        private readonly IIOService _ioService;
        private readonly ILogService _logService;
        private readonly IRemoteControlService _remoteControlService;
        private readonly IPatchService _patchService;
        private readonly IDatabaseFactoryService _databaseFactoryService;
        private readonly string _localSettingsDirectory;
        private readonly Func<Exception, bool> _reportAndSwallowException;

        private void LogInfo(string text) => _logService.LogInfo(text);

        private void LogException(Exception e, string text) => _logService.LogException(e, text);

        internal void RegisterLanguage(string languageName)
        {
            this.AssertIsForeground();
            _registeredLanguageNames.Add(languageName);
            OnOptionChanged(null, EventArgs.Empty);
        }

        private void OnOptionChanged(object sender, EventArgs e)
        {
            // If we don't have any add-import features that would use these indices, then
            // don't bother creating them.
            if (!_registeredLanguageNames.Any(IsRegisteredForLanguage))
            {
                return;
            }

            // Kick off a database update.  Wait a few seconds before starting so we don't
            // interfere too much with solution loading.
            var sources = _installerService.PackageSources;

            // Always pull down the nuget.org index.  It contains the MS reference assembly index
            // inside of it.
            var allSources = sources.Concat(new PackageSource(NugetOrgSource, source: null));
            foreach (var source in allSources)
            {
                Task.Delay(TimeSpan.FromSeconds(10)).ContinueWith(_ =>
                    UpdateSourceInBackgroundAsync(source.Name), TaskScheduler.Default);
            }<commit_msg>Remove unused method.<commit_after>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Linq;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.Elfie.Model;
using Microsoft.CodeAnalysis.Packaging;
using Microsoft.CodeAnalysis.Shared.Options;
using Microsoft.CodeAnalysis.Shared.Utilities;
using Microsoft.Internal.VisualStudio.Shell.Interop;
using Roslyn.Utilities;
using static System.FormattableString;

namespace Microsoft.VisualStudio.LanguageServices.SymbolSearch
{
    /// <summary>
    /// A service which enables searching for packages matching certain criteria.
    /// It works against an <see cref="Microsoft.CodeAnalysis.Elfie"/> database to find results.
    /// 
    /// This implementation also spawns a task which will attempt to keep that database up to
    /// date by downloading patches on a daily basis.
    /// </summary>
    internal partial class SymbolSearchService
    {
        // Internal for testing purposes.
        internal const string ContentAttributeName = "content";
        internal const string ChecksumAttributeName = "checksum";
        internal const string UpToDateAttributeName = "upToDate";
        internal const string TooOldAttributeName = "tooOld";
        internal const string NugetOrgSource = "nuget.org";

        private const string HostId = "RoslynNuGetSearch";
        private const string MicrosoftAssemblyReferencesName = "MicrosoftAssemblyReferences";
        private static readonly LinkedList<string> s_log = new LinkedList<string>();

        /// <summary>
        /// Cancellation support for the task we use to keep the local database up to date.
        /// When VS shuts down it will dispose us.  We'll cancel the task at that point.
        /// </summary>
        private readonly CancellationTokenSource _cancellationTokenSource;
        private readonly CancellationToken _cancellationToken;

        private readonly ConcurrentDictionary<string, object> _sourceToUpdateSentinel =
            new ConcurrentDictionary<string, object>();

        // Interfaces that abstract out the external functionality we need.  Used so we can easily
        // mock behavior during tests.
        private readonly IPackageInstallerService _installerService;
        private readonly IDelayService _delayService;
        private readonly IIOService _ioService;
        private readonly ILogService _logService;
        private readonly IRemoteControlService _remoteControlService;
        private readonly IPatchService _patchService;
        private readonly IDatabaseFactoryService _databaseFactoryService;
        private readonly string _localSettingsDirectory;
        private readonly Func<Exception, bool> _reportAndSwallowException;

        private void LogInfo(string text) => _logService.LogInfo(text);

        private void LogException(Exception e, string text) => _logService.LogException(e, text);

        private void OnOptionChanged(object sender, EventArgs e)
        {
            // If we don't have any add-import features that would use these indices, then
            // don't bother creating them.
            if (!_registeredLanguageNames.Any(IsRegisteredForLanguage))
            {
                return;
            }

            // Kick off a database update.  Wait a few seconds before starting so we don't
            // interfere too much with solution loading.
            var sources = _installerService.PackageSources;

            // Always pull down the nuget.org index.  It contains the MS reference assembly index
            // inside of it.
            var allSources = sources.Concat(new PackageSource(NugetOrgSource, source: null));
            foreach (var source in allSources)
            {
                Task.Delay(TimeSpan.FromSeconds(10)).ContinueWith(_ =>
                    UpdateSourceInBackgroundAsync(source.Name), TaskScheduler.Default);
            }
{'fraction_non_alphanumeric': 0.06812485759854181, 'fraction_numerical': 0.000911369332421964, 'mean_word_length': 2.9940855323020927, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2497754', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2028, 'n_words': 722}