<commit_before>﻿// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.ErrorReporting;
using Microsoft.CodeAnalysis.Host;
using Microsoft.CodeAnalysis.Options;
using Microsoft.CodeAnalysis.Shared.Utilities;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.Storage
{
    /// <summary>
    /// A service that enables storing and retrieving of information associated with solutions,
    /// projects or documents across runtime sessions.
    /// </summary>
    internal abstract partial class AbstractPersistentStorageService : IChecksummedPersistentStorageService
    {
        private readonly IOptionService _optionService;
        private readonly IPersistentStorageLocationService _locationService;

        /// <summary>
        /// This lock guards all mutable fields in this type.
        /// </summary>
        private readonly object _lock = new object();
        private ReferenceCountedDisposable<IChecksummedPersistentStorage> _currentPersistentStorage;
        private SolutionId _currentPersistentStorageSolutionId;

        protected AbstractPersistentStorageService(
            IOptionService optionService,
            IPersistentStorageLocationService locationService)
        {
            _optionService = optionService;
            _locationService = locationService;
        }

        protected abstract string GetDatabaseFilePath(string workingFolderPath);
        protected abstract bool TryOpenDatabase(Solution solution, string workingFolderPath, string databaseFilePath, out IChecksummedPersistentStorage storage);
        protected abstract bool ShouldDeleteDatabase(Exception exception);

        IPersistentStorage IPersistentStorageService.GetStorage(Solution solution)
            => this.GetStorage(solution);

        IPersistentStorage IPersistentStorageService2.GetStorage(Solution solution, bool checkBranchId)
            => this.GetStorage(solution, checkBranchId);

        public IChecksummedPersistentStorage GetStorage(Solution solution)
            => GetStorage(solution, checkBranchId: true);

        public IChecksummedPersistentStorage GetStorage(Solution solution, bool checkBranchId)
        {
            if (!DatabaseSupported(solution, checkBranchId))
            {
                return NoOpPersistentStorage.Instance;
            }

            return GetStorageWorker(solution);
        }

        internal IChecksummedPersistentStorage GetStorageWorker(Solution solution)
        {
            lock (_lock)
            {
                // Do we already have storage for this?
                if (solution.Id == _currentPersistentStorageSolutionId)
                {
                    // We do, great
                    return PersistentStorageReferenceCountedDisposableWrapper.AddReferenceCountToAndCreateWrapper(_currentPersistentStorage);
                }

                var workingFolder = _locationService.TryGetStorageLocation(solution);
                if (workingFolder == null)
                {
                    return NoOpPersistentStorage.Instance;
                }

                // If we already had some previous cached service, let's let it start cleaning up
                if (_currentPersistentStorage != null)
                {
                    var storageToDispose = _currentPersistentStorage;

                    Task.Run(() => storageToDispose.Dispose());

                    _currentPersistentStorage = null;
                    _currentPersistentStorageSolutionId = null;
                }

                _currentPersistentStorage = TryCreatePersistentStorage(solution, workingFolder);

                if (_currentPersistentStorage == null)
                {
                    return NoOpPersistentStorage.Instance;
                }

                _currentPersistentStorageSolutionId = solution.Id;

                return PersistentStorageReferenceCountedDisposableWrapper.AddReferenceCountToAndCreateWrapper(_currentPersistentStorage);
            }
        }

        private bool DatabaseSupported(Solution solution, bool checkBranchId)
        {
            if (solution.FilePath == null)
            {
                return false;
            }

            if (checkBranchId && solution.BranchId != solution.Workspace.PrimaryBranchId)
            {
                // we only use database for primary solution. (Ex, forked solution will not use database)
                return false;
            }

            return true;
        }

        private ReferenceCountedDisposable<IChecksummedPersistentStorage> TryCreatePersistentStorage(Solution solution, string workingFolderPath)
        {
            // Attempt to create the database up to two times.  The first time we may encounter
            // some sort of issue (like DB corruption).  We'll then try to delete the DB and can
            // try to create it again.  If we can't create it the second time, then there's nothing
            // we can do and we have to store things in memory.
            if (TryCreatePersistentStorage(solution, workingFolderPath, out var persistentStorage) ||
                TryCreatePersistentStorage(solution, workingFolderPath, out persistentStorage))
            {
                return new ReferenceCountedDisposable<IChecksummedPersistentStorage>(persistentStorage);
            }

            // okay, can't recover, then use no op persistent service 
            // so that things works old way (cache everything in memory)
            return null;
        }

        private bool TryCreatePersistentStorage(
            Solution solution,
            string workingFolderPath,
            out IChecksummedPersistentStorage persistentStorage)
        {
            persistentStorage = null;

            var databaseFilePath = GetDatabaseFilePath(workingFolderPath);
            try
            {
                if (!TryOpenDatabase(solution, workingFolderPath, databaseFilePath, out persistentStorage))
                {
                    return false;
                }

                return true;
            }
            catch (Exception ex)
            {
                StorageDatabaseLogger.LogException(ex);

                if (ShouldDeleteDatabase(ex))
                {
                    // this was not a normal exception that we expected during DB open.
                    // Report this so we can try to address whatever is causing this.
                    FatalError.ReportWithoutCrash(ex);
                    IOUtilities.PerformIO(() => Directory.Delete(Path.GetDirectoryName(databaseFilePath), recursive: true));
                }

                return false;
            }
        }

        private void Shutdown()
        {
            ReferenceCountedDisposable<IChecksummedPersistentStorage> storage = null;

            lock (_lock)
            {
                // We will transfer ownership in a thread-safe way out so we can dispose outside the lock
                storage = _currentPersistentStorage;
                _currentPersistentStorage = null;
                _currentPersistentStorageSolutionId = null;
            }

            if (storage != null)
            {
                // Dispose storage outside of the lock. Note this only removes our reference count; clients who are still
                // using this will still be holding a reference count.
                storage.Dispose();
            }
        }

        internal TestAccessor GetTestAccessor()
            => new TestAccessor(this);

        internal readonly struct TestAccessor
        {
            private readonly AbstractPersistentStorageService _service;

            public TestAccessor(AbstractPersistentStorageService service)<commit_msg>If we fail to create the sqlite db, don't continually retry for every client that wants it.<commit_after>﻿// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.ErrorReporting;
using Microsoft.CodeAnalysis.Host;
using Microsoft.CodeAnalysis.Options;
using Microsoft.CodeAnalysis.Shared.Utilities;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.Storage
{
    /// <summary>
    /// A service that enables storing and retrieving of information associated with solutions,
    /// projects or documents across runtime sessions.
    /// </summary>
    internal abstract partial class AbstractPersistentStorageService : IChecksummedPersistentStorageService
    {
        private readonly IPersistentStorageLocationService _locationService;

        /// <summary>
        /// This lock guards all mutable fields in this type.
        /// </summary>
        private readonly object _lock = new object();
        private ReferenceCountedDisposable<IChecksummedPersistentStorage> _currentPersistentStorage;
        private SolutionId _currentPersistentStorageSolutionId;

        protected AbstractPersistentStorageService(IPersistentStorageLocationService locationService)
            => _locationService = locationService;

        protected abstract string GetDatabaseFilePath(string workingFolderPath);
        protected abstract bool TryOpenDatabase(Solution solution, string workingFolderPath, string databaseFilePath, out IChecksummedPersistentStorage storage);
        protected abstract bool ShouldDeleteDatabase(Exception exception);

        IPersistentStorage IPersistentStorageService.GetStorage(Solution solution)
            => this.GetStorage(solution);

        IPersistentStorage IPersistentStorageService2.GetStorage(Solution solution, bool checkBranchId)
            => this.GetStorage(solution, checkBranchId);

        public IChecksummedPersistentStorage GetStorage(Solution solution)
            => GetStorage(solution, checkBranchId: true);

        public IChecksummedPersistentStorage GetStorage(Solution solution, bool checkBranchId)
        {
            if (!DatabaseSupported(solution, checkBranchId))
            {
                return NoOpPersistentStorage.Instance;
            }

            return GetStorageWorker(solution);
        }

        internal IChecksummedPersistentStorage GetStorageWorker(Solution solution)
        {
            lock (_lock)
            {
                // Do we already have storage for this?
                if (solution.Id == _currentPersistentStorageSolutionId)
                {
                    // We do, great up our ref count for our caller.  They'll decrement it when done
                    // with it.
                    return PersistentStorageReferenceCountedDisposableWrapper.AddReferenceCountToAndCreateWrapper(_currentPersistentStorage);
                }

                var workingFolder = _locationService.TryGetStorageLocation(solution);
                if (workingFolder == null)
                {
                    return NoOpPersistentStorage.Instance;
                }

                // If we already had some previous cached service, let's let it start cleaning up
                if (_currentPersistentStorage != null)
                {
                    var storageToDispose = _currentPersistentStorage;

                    // Kick off a task to actually go dispose the previous cached storage instance.
                    // This will remove the single ref count we ourselves added when we cached the
                    // instance.  Then once all other existing clients who are holding onto this
                    // instance let go, it will finally get truly disposed.
                    Task.Run(() => storageToDispose.Dispose());

                    _currentPersistentStorage = null;
                    _currentPersistentStorageSolutionId = null;
                }

                var storage = CreatePersistentStorage(solution, workingFolder);
                Contract.ThrowIfNull(storage);

                // Create and cache a new storage instance associated with this particular solution.
                // It will initially have a ref-count of 1 due to our reference to it.
                _currentPersistentStorage = new ReferenceCountedDisposable<IChecksummedPersistentStorage>(storage);
                _currentPersistentStorageSolutionId = solution.Id;

                // Now increment the reference count and return to our caller.  The current ref
                // count for this instance will be 2.  Until all the callers *and* us decrement
                // the refcounts, this instance will not be actually disposed.
                return PersistentStorageReferenceCountedDisposableWrapper.AddReferenceCountToAndCreateWrapper(_currentPersistentStorage);
            }
        }

        private bool DatabaseSupported(Solution solution, bool checkBranchId)
        {
            if (solution.FilePath == null)
            {
                return false;
            }

            if (checkBranchId && solution.BranchId != solution.Workspace.PrimaryBranchId)
            {
                // we only use database for primary solution. (Ex, forked solution will not use database)
                return false;
            }

            return true;
        }

        private IChecksummedPersistentStorage CreatePersistentStorage(Solution solution, string workingFolderPath)
        {
            // Attempt to create the database up to two times.  The first time we may encounter
            // some sort of issue (like DB corruption).  We'll then try to delete the DB and can
            // try to create it again.  If we can't create it the second time, then there's nothing
            // we can do and we have to store things in memory.
            return TryCreatePersistentStorage(solution, workingFolderPath) ??
                   TryCreatePersistentStorage(solution, workingFolderPath) ??
                   NoOpPersistentStorage.Instance;
        }

        private IChecksummedPersistentStorage? TryCreatePersistentStorage(
            Solution solution,
            string workingFolderPath)
        {
            var databaseFilePath = GetDatabaseFilePath(workingFolderPath);
            try
            {
                return TryOpenDatabase(solution, workingFolderPath, databaseFilePath, out var persistentStorage)
                    ? persistentStorage
                    : null;
            }
            catch (Exception ex)
            {
                StorageDatabaseLogger.LogException(ex);

                if (ShouldDeleteDatabase(ex))
                {
                    // this was not a normal exception that we expected during DB open.
                    // Report this so we can try to address whatever is causing this.
                    FatalError.ReportWithoutCrash(ex);
                    IOUtilities.PerformIO(() => Directory.Delete(Path.GetDirectoryName(databaseFilePath), recursive: true));
                }

                return null;
            }
        }

        private void Shutdown()
        {
            ReferenceCountedDisposable<IChecksummedPersistentStorage> storage = null;

            lock (_lock)
            {
                // We will transfer ownership in a thread-safe way out so we can dispose outside the lock
                storage = _currentPersistentStorage;
                _currentPersistentStorage = null;
                _currentPersistentStorageSolutionId = null;
            }

            if (storage != null)
            {
                // Dispose storage outside of the lock. Note this only removes our reference count; clients who are still
                // using this will still be holding a reference count.
                storage.Dispose();
            }
        }

        internal TestAccessor GetTestAccessor()
            => new TestAccessor(this);

        internal readonly struct TestAccessor
        {
            private readonly AbstractPersistentStorageService _service;

            public TestAccessor(AbstractPersistentStorageService service)
{'fraction_non_alphanumeric': 0.054915420446187795, 'fraction_numerical': 0.00024515812699190976, 'mean_word_length': 2.0533308383233533, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6441774', 'n_tokens_mistral': 3624, 'n_tokens_neox': 3405, 'n_words': 1118}