<commit_before>﻿// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Common.Core;
using Microsoft.Common.Core.Shell;
using Microsoft.Common.Core.Threading;
using Microsoft.Languages.Editor.Tasks;
using Microsoft.R.Components.InteractiveWorkflow;
using Microsoft.R.Components.PackageManager.Model;
using Microsoft.R.Host.Client;
using Microsoft.R.Host.Client.Host;
using Microsoft.R.Host.Client.Session;
using Newtonsoft.Json.Linq;
using static System.FormattableString;

namespace Microsoft.R.Support.Help.Packages {
    /// <summary>
    /// Index of packages available from the R engine.
    /// </summary>
    [Export(typeof(IPackageIndex))]
    public sealed class PackageIndex : IPackageIndex {
        private readonly IRSession _interactiveSession;
        private readonly ICoreShell _shell;
        private readonly IIntellisenseRSession _host;
        private readonly IFunctionIndex _functionIndex;
        private readonly ConcurrentDictionary<string, PackageInfo> _packages = new ConcurrentDictionary<string, PackageInfo>();
        private readonly BinaryAsyncLock _buildIndexLock = new BinaryAsyncLock();

        public static IEnumerable<string> PreloadedPackages { get; } = new string[]
            { "base", "stats", "utils", "graphics", "datasets", "methods" };

        [ImportingConstructor]
        public PackageIndex(IRInteractiveWorkflowProvider interactiveWorkflowProvider, ICoreShell shell, IIntellisenseRSession host, IFunctionIndex functionIndex) {
            _shell = shell;
            _host = host;
            _functionIndex = functionIndex;

            _interactiveSession = interactiveWorkflowProvider.GetOrCreate().RSession;

            _interactiveSession.PackagesInstalled += OnPackagesChanged;
            _interactiveSession.PackagesRemoved += OnPackagesChanged;
        }

        private void OnPackagesChanged(object sender, EventArgs e) => ScheduleIdleTimeRebuild();

        #region IPackageIndex
        /// <summary>
        /// Collection of all packages (base, user and project-specific)
        /// </summary>
        public IEnumerable<IPackageInfo> Packages => _packages.Values;

        public async Task BuildIndexAsync() {
            var lockToken = await _buildIndexLock.WaitAsync();
            await BuildIndexAsync(lockToken);
        }

        private async Task BuildIndexAsync(IBinaryAsyncLockToken lockToken) {
            try {
                if (!lockToken.IsSet) {
                    var startTotalTime = DateTime.Now;

                    await TaskUtilities.SwitchToBackgroundThread();
                    await _host.CreateSessionAsync();
                    Debug.WriteLine("R function host start: {0} ms", (DateTime.Now - startTotalTime).TotalMilliseconds);

                    var startTime = DateTime.Now;
                    // Fetch list of available packages from R session
                    await BuildPackageListAsync();
                    Debug.WriteLine("R package names/description: {0} ms", (DateTime.Now - startTime).TotalMilliseconds);

                    // Populate function index for preloaded packages first
                    startTime = DateTime.Now;
                    await BuildPreloadedPackagesFunctionListAsync();
                    Debug.WriteLine("R function index (preloaded): {0} ms", (DateTime.Now - startTime).TotalMilliseconds);

                    // Populate function index for all remaining packages
                    startTime = DateTime.Now;
                    await BuildRemainingPackagesFunctionListAsync();
                    Debug.WriteLine("R function index (remaining): {0} ms", (DateTime.Now - startTime).TotalMilliseconds);

                    await _functionIndex.BuildIndexAsync(this);
                    Debug.WriteLine("R function index total: {0} ms", (DateTime.Now - startTotalTime).TotalMilliseconds);
                }
            } catch (RHostDisconnectedException ex) {
                Debug.WriteLine(ex.Message);
            } finally {
                lockToken.Set();
            }
        }

        /// <summary>
        /// Retrieves information on the package from index. Does not attempt to locate the package
        /// if it is not in the index such as when package was just installed.
        /// </summary>
        public IPackageInfo GetPackageInfo(string packageName) {
            PackageInfo package = null;
            packageName = packageName.TrimQuotes().Trim();
            _packages.TryGetValue(packageName, out package);
            return package;
        }

        /// <summary>
        /// Retrieves R package information by name. If package is not in the index,
        /// attempts to locate the package in the current R session.
        /// </summary>
        public Task<IPackageInfo> GetPackageInfoAsync(string packageName) {
            packageName = packageName.TrimQuotes().Trim();
            IPackageInfo package = GetPackageInfo(packageName);
            if (package == null) {
                return TryAddNewPackageAsync(packageName);
            }
            return Task.FromResult(package);
        }

        public void WriteToDisk() {
            if (_buildIndexLock.IsSet) {
                foreach (var pi in _packages.Values) {
                    pi.WriteToDisk();
                }
            }
        }
        #endregion

        public void Dispose() {
            if (_interactiveSession != null) {
                _interactiveSession.PackagesInstalled -= OnPackagesChanged;
                _interactiveSession.PackagesRemoved -= OnPackagesChanged;
            }
            _host?.Dispose();
        }

        private async Task BuildPackageListAsync() {
            var packages = await GetPackagesAsync();
            foreach (var p in packages) {
                _packages[p.Package] = new PackageInfo(_host, p.Package, p.Description, p.Version);
            }
        }

        private async Task BuildPreloadedPackagesFunctionListAsync() {
            foreach (var packageName in PreloadedPackages) {
                PackageInfo pi;
                _packages.TryGetValue(packageName, out pi);
                if (pi != null) {
                    await pi.LoadFunctionsIndexAsync();
                }
            }
        }

        private async Task BuildRemainingPackagesFunctionListAsync() {
            foreach (var pi in _packages.Values) {
                if (!pi.Functions.Any()) {
                    await pi.LoadFunctionsIndexAsync();
                }
            }
        }

        private async Task<IPackageInfo> TryAddNewPackageAsync(string packageName) {
            PackageInfo packageInfo = null;
            try {
                if (packageName.EqualsOrdinal("rtvs")) {
                    packageInfo = new PackageInfo(_host, packageName, "R Tools", "1.0");
                } else {
                    var packages = await GetPackagesAsync();
                    var package = packages.FirstOrDefault(p => p.Package.EqualsOrdinal(packageName));
                    if (package != null) {
                        packageInfo = new PackageInfo(_host, package.Package, package.Description, package.Version);
                    }
                }

                if (packageInfo != null) {
                    await packageInfo.LoadFunctionsIndexAsync();
                    _packages[packageName] = packageInfo;
                    _functionIndex.RegisterPackageFunctions(packageInfo);
                }
            } catch (RHostDisconnectedException) { }
            return packageInfo;
        }

        private async Task<IEnumerable<RPackage>> GetPackagesAsync() {
            try {
                await _host.CreateSessionAsync();
                var result = await _host.Session.InstalledPackagesAsync();
                return result.Select(p => p.ToObject<RPackage>());
            } catch (TaskCanceledException) { }<commit_msg>Add rtvs to package list completion<commit_after>﻿// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Common.Core;
using Microsoft.Common.Core.Shell;
using Microsoft.Common.Core.Threading;
using Microsoft.Languages.Editor.Tasks;
using Microsoft.R.Components.InteractiveWorkflow;
using Microsoft.R.Components.PackageManager.Model;
using Microsoft.R.Host.Client;
using Microsoft.R.Host.Client.Host;
using Microsoft.R.Host.Client.Session;
using Newtonsoft.Json.Linq;
using static System.FormattableString;

namespace Microsoft.R.Support.Help.Packages {
    /// <summary>
    /// Index of packages available from the R engine.
    /// </summary>
    [Export(typeof(IPackageIndex))]
    public sealed class PackageIndex : IPackageIndex {
        private readonly IRSession _interactiveSession;
        private readonly ICoreShell _shell;
        private readonly IIntellisenseRSession _host;
        private readonly IFunctionIndex _functionIndex;
        private readonly ConcurrentDictionary<string, PackageInfo> _packages = new ConcurrentDictionary<string, PackageInfo>();
        private readonly BinaryAsyncLock _buildIndexLock = new BinaryAsyncLock();

        public static IEnumerable<string> PreloadedPackages { get; } = new string[]
            { "base", "stats", "utils", "graphics", "datasets", "methods" };

        [ImportingConstructor]
        public PackageIndex(IRInteractiveWorkflowProvider interactiveWorkflowProvider, ICoreShell shell, IIntellisenseRSession host, IFunctionIndex functionIndex) {
            _shell = shell;
            _host = host;
            _functionIndex = functionIndex;

            _interactiveSession = interactiveWorkflowProvider.GetOrCreate().RSession;

            _interactiveSession.PackagesInstalled += OnPackagesChanged;
            _interactiveSession.PackagesRemoved += OnPackagesChanged;
        }

        private void OnPackagesChanged(object sender, EventArgs e) => ScheduleIdleTimeRebuild();

        #region IPackageIndex
        /// <summary>
        /// Collection of all packages (base, user and project-specific)
        /// </summary>
        public IEnumerable<IPackageInfo> Packages => _packages.Values;

        public async Task BuildIndexAsync() {
            var lockToken = await _buildIndexLock.WaitAsync();
            await BuildIndexAsync(lockToken);
        }

        private async Task BuildIndexAsync(IBinaryAsyncLockToken lockToken) {
            try {
                if (!lockToken.IsSet) {
                    var startTotalTime = DateTime.Now;

                    await TaskUtilities.SwitchToBackgroundThread();
                    await _host.CreateSessionAsync();
                    Debug.WriteLine("R function host start: {0} ms", (DateTime.Now - startTotalTime).TotalMilliseconds);

                    var startTime = DateTime.Now;
                    // Fetch list of available packages from R session
                    await BuildPackageListAsync();
                    Debug.WriteLine("R package names/description: {0} ms", (DateTime.Now - startTime).TotalMilliseconds);

                    // Populate function index for preloaded packages first
                    startTime = DateTime.Now;
                    await BuildPreloadedPackagesFunctionListAsync();
                    Debug.WriteLine("R function index (preloaded): {0} ms", (DateTime.Now - startTime).TotalMilliseconds);

                    // Populate function index for all remaining packages
                    startTime = DateTime.Now;
                    await BuildRemainingPackagesFunctionListAsync();
                    Debug.WriteLine("R function index (remaining): {0} ms", (DateTime.Now - startTime).TotalMilliseconds);

                    await _functionIndex.BuildIndexAsync(this);
                    Debug.WriteLine("R function index total: {0} ms", (DateTime.Now - startTotalTime).TotalMilliseconds);
                }
            } catch (RHostDisconnectedException ex) {
                Debug.WriteLine(ex.Message);
            } finally {
                lockToken.Set();
            }
        }

        /// <summary>
        /// Retrieves information on the package from index. Does not attempt to locate the package
        /// if it is not in the index such as when package was just installed.
        /// </summary>
        public IPackageInfo GetPackageInfo(string packageName) {
            PackageInfo package = null;
            packageName = packageName.TrimQuotes().Trim();
            _packages.TryGetValue(packageName, out package);
            return package;
        }

        /// <summary>
        /// Retrieves R package information by name. If package is not in the index,
        /// attempts to locate the package in the current R session.
        /// </summary>
        public Task<IPackageInfo> GetPackageInfoAsync(string packageName) {
            packageName = packageName.TrimQuotes().Trim();
            IPackageInfo package = GetPackageInfo(packageName);
            if (package == null) {
                return TryAddNewPackageAsync(packageName);
            }
            return Task.FromResult(package);
        }

        public void WriteToDisk() {
            if (_buildIndexLock.IsSet) {
                foreach (var pi in _packages.Values) {
                    pi.WriteToDisk();
                }
            }
        }
        #endregion

        public void Dispose() {
            if (_interactiveSession != null) {
                _interactiveSession.PackagesInstalled -= OnPackagesChanged;
                _interactiveSession.PackagesRemoved -= OnPackagesChanged;
            }
            _host?.Dispose();
        }

        private async Task BuildPackageListAsync() {
            var packages = await GetPackagesAsync();
            foreach (var p in packages) {
                _packages[p.Package] = new PackageInfo(_host, p.Package, p.Description, p.Version);
            }
            _packages["rtvs"] = new PackageInfo(_host, "rtvs", "R Tools", "1.0");
        }

        private async Task BuildPreloadedPackagesFunctionListAsync() {
            foreach (var packageName in PreloadedPackages) {
                PackageInfo pi;
                _packages.TryGetValue(packageName, out pi);
                if (pi != null) {
                    await pi.LoadFunctionsIndexAsync();
                }
            }
        }

        private async Task BuildRemainingPackagesFunctionListAsync() {
            foreach (var pi in _packages.Values) {
                if (!pi.Functions.Any()) {
                    await pi.LoadFunctionsIndexAsync();
                }
            }
        }

        private async Task<IPackageInfo> TryAddNewPackageAsync(string packageName) {
            try {
                var packages = await GetPackagesAsync();
                var package = packages.FirstOrDefault(p => p.Package.EqualsOrdinal(packageName));
                if (package != null) {
                    var p = new PackageInfo(_host, package.Package, package.Description, package.Version);
                    await p.LoadFunctionsIndexAsync();
                    _packages[packageName] = p;
                    _functionIndex.RegisterPackageFunctions(p);
                    return p;
                }
            } catch (RHostDisconnectedException) { }
            return null;
        }

        private async Task<IEnumerable<RPackage>> GetPackagesAsync() {
            try {
                await _host.CreateSessionAsync();
                var result = await _host.Session.InstalledPackagesAsync();
                return result.Select(p => p.ToObject<RPackage>());
            } catch (TaskCanceledException) { }
{'fraction_non_alphanumeric': 0.08488587291795188, 'fraction_numerical': 0.0008636644046884639, 'mean_word_length': 2.064461247637051, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7143284', 'n_tokens_mistral': 3889, 'n_tokens_neox': 3769, 'n_words': 993}