<commit_before>// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using Microsoft.CodeAnalysis.Editor.Tagging;
using Microsoft.CodeAnalysis.Shared.TestHooks;
using Microsoft.VisualStudio.Text;
using Microsoft.VisualStudio.Text.Tagging;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.Editor.Shared.Tagging
{
    /// <summary>
    /// Async thin tagger implementation.
    ///
    /// Actual tag information is stored in TagSource and shared between multiple taggers created 
    /// for same views or buffers.
    ///
    /// It's responsibility is on interaction between host and tagger. TagSource has responsibility 
    /// on how to provide information for this tagger.
    /// </summary>
    internal sealed partial class AsynchronousTagger<TTag> : IAccurateTagger<TTag>, IDisposable
        where TTag : ITag
    {
        private const int MaxNumberOfRequestedSpans = 100;

        #region Fields that can be accessed from either thread

        private readonly ITextBuffer _subjectBuffer;

        // Internal for testing purposes.
        internal readonly TagSource<TTag> TagSource;

        #endregion

        #region Fields that can only be accessed from the foreground thread

        /// <summary>
        /// The batch change notifier that we use to throttle update to the UI.
        /// </summary>
        private readonly BatchChangeNotifier _batchChangeNotifier;

        #endregion

        public event EventHandler<SnapshotSpanEventArgs> TagsChanged;

        public AsynchronousTagger(
            IAsynchronousOperationListener listener,
            IForegroundNotificationService notificationService,
            TagSource<TTag> tagSource,
            ITextBuffer subjectBuffer)
        {
            Contract.ThrowIfNull(subjectBuffer);

            _subjectBuffer = subjectBuffer;
            _batchChangeNotifier = new BatchChangeNotifier(subjectBuffer, listener, notificationService, ReportChangedSpan);

            TagSource = tagSource;

            TagSource.OnTaggerAdded(this);
            TagSource.TagsChangedForBuffer += OnTagsChangedForBuffer;
            TagSource.Paused += OnPaused;
            TagSource.Resumed += OnResumed;
        }

        public void Dispose()
        {
            TagSource.Resumed -= OnResumed;
            TagSource.Paused -= OnPaused;
            TagSource.TagsChangedForBuffer -= OnTagsChangedForBuffer;
            TagSource.OnTaggerDisposed(this);
        }

        private void ReportChangedSpan(SnapshotSpan changeSpan)
        {
            var tagsChanged = TagsChanged;
            if (tagsChanged != null)
            {
                tagsChanged(this, new SnapshotSpanEventArgs(changeSpan));
            }
        }

        private void OnPaused(object sender, EventArgs e)
        {
            _batchChangeNotifier.Pause();
        }

        private void OnResumed(object sender, EventArgs e)
        {
            _batchChangeNotifier.Resume();
        }

        private void OnTagsChangedForBuffer(ICollection<KeyValuePair<ITextBuffer,NormalizedSnapshotSpanCollection>> changes)
        {
            // Note: This operation is uncancellable. Once we've been notified here, our cached tags
            // in the tag source are new. If we don't update the UI of the editor then we will end
            // up in an inconsistent state between us and the editor where we have new tags but the
            // editor will never know.

            TagSource.RegisterNotification(() =>
            {
                foreach (var change in changes)
                {
                    if (change.Key != _subjectBuffer)
                    {
                        continue;
                    }

                    // Now report them back to the UI on the main thread.
                    _batchChangeNotifier.EnqueueChanges(change.Value);
                }

            }, TaggerDelay.NearImmediate.ComputeTimeDelayMS(), CancellationToken.None);
        }

        public IEnumerable<ITagSpan<TTag>> GetTags(NormalizedSnapshotSpanCollection requestedSpans)
        {
            return GetTagsWorker(requestedSpans, accurate: false, cancellationToken: CancellationToken.None);
        }

        public IEnumerable<ITagSpan<TTag>> GetAllTags(NormalizedSnapshotSpanCollection requestedSpans, CancellationToken cancellationToken)
        {
            return GetTagsWorker(requestedSpans, accurate: true, cancellationToken: cancellationToken);
        }

        private IEnumerable<ITagSpan<TTag>> GetTagsWorker(
            NormalizedSnapshotSpanCollection requestedSpans,
            bool accurate,
            CancellationToken cancellationToken)
        {
            if (requestedSpans.Count == 0)
            {
                return SpecializedCollections.EmptyEnumerable<ITagSpan<TTag>>();
            }

            var buffer = requestedSpans.First().Snapshot.TextBuffer;
            var tags = accurate
                ? TagSource.GetAccurateTagIntervalTreeForBuffer(buffer, cancellationToken)
                : TagSource.GetTagIntervalTreeForBuffer(buffer);

            if (tags == null)
            {
                return SpecializedCollections.EmptyEnumerable<ITagSpan<TTag>>();
            }

            var result = GetTags(requestedSpans, tags);

            DebugVerifyTags(requestedSpans, result);

            return result;
        }

        private static IEnumerable<ITagSpan<TTag>> GetTags(NormalizedSnapshotSpanCollection requestedSpans, ITagSpanIntervalTree<TTag> tags)
        {
            // Special case the case where there is only one requested span.  In that case, we don't
            // need to allocate any intermediate collections
            return requestedSpans.Count == 1
                ? tags.GetIntersectingSpans(requestedSpans[0])
                : requestedSpans.Count < MaxNumberOfRequestedSpans
                    ? GetTagsForSmallNumberOfSpans(requestedSpans, tags)
                    : GetTagsForLargeNumberOfSpans(requestedSpans, tags);
        }

        private static IEnumerable<ITagSpan<TTag>> GetTagsForSmallNumberOfSpans(
            NormalizedSnapshotSpanCollection requestedSpans,
            ITagSpanIntervalTree<TTag> tags)
        {
            var result = new List<ITagSpan<TTag>>();

            foreach (var s in requestedSpans)<commit_msg>Don't expose TagSource.  It's an implementation detail.<commit_after>// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using Microsoft.CodeAnalysis.Editor.Tagging;
using Microsoft.CodeAnalysis.Shared.TestHooks;
using Microsoft.VisualStudio.Text;
using Microsoft.VisualStudio.Text.Tagging;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.Editor.Shared.Tagging
{
    /// <summary>
    /// Async thin tagger implementation.
    ///
    /// Actual tag information is stored in TagSource and shared between multiple taggers created 
    /// for same views or buffers.
    ///
    /// It's responsibility is on interaction between host and tagger. TagSource has responsibility 
    /// on how to provide information for this tagger.
    /// </summary>
    internal sealed partial class AsynchronousTagger<TTag> : IAccurateTagger<TTag>, IDisposable
        where TTag : ITag
    {
        private const int MaxNumberOfRequestedSpans = 100;

        #region Fields that can be accessed from either thread

        private readonly ITextBuffer _subjectBuffer;

        private readonly TagSource<TTag> _tagSource;

        #endregion

        #region Fields that can only be accessed from the foreground thread

        /// <summary>
        /// The batch change notifier that we use to throttle update to the UI.
        /// </summary>
        private readonly BatchChangeNotifier _batchChangeNotifier;

        #endregion

        public event EventHandler<SnapshotSpanEventArgs> TagsChanged;

        public AsynchronousTagger(
            IAsynchronousOperationListener listener,
            IForegroundNotificationService notificationService,
            TagSource<TTag> tagSource,
            ITextBuffer subjectBuffer)
        {
            Contract.ThrowIfNull(subjectBuffer);

            _subjectBuffer = subjectBuffer;
            _batchChangeNotifier = new BatchChangeNotifier(subjectBuffer, listener, notificationService, ReportChangedSpan);

            _tagSource = tagSource;

            _tagSource.OnTaggerAdded(this);
            _tagSource.TagsChangedForBuffer += OnTagsChangedForBuffer;
            _tagSource.Paused += OnPaused;
            _tagSource.Resumed += OnResumed;
        }

        public void Dispose()
        {
            _tagSource.Resumed -= OnResumed;
            _tagSource.Paused -= OnPaused;
            _tagSource.TagsChangedForBuffer -= OnTagsChangedForBuffer;
            _tagSource.OnTaggerDisposed(this);
        }

        private void ReportChangedSpan(SnapshotSpan changeSpan)
        {
            var tagsChanged = TagsChanged;
            if (tagsChanged != null)
            {
                tagsChanged(this, new SnapshotSpanEventArgs(changeSpan));
            }
        }

        private void OnPaused(object sender, EventArgs e)
        {
            _batchChangeNotifier.Pause();
        }

        private void OnResumed(object sender, EventArgs e)
        {
            _batchChangeNotifier.Resume();
        }

        private void OnTagsChangedForBuffer(ICollection<KeyValuePair<ITextBuffer,NormalizedSnapshotSpanCollection>> changes)
        {
            // Note: This operation is uncancellable. Once we've been notified here, our cached tags
            // in the tag source are new. If we don't update the UI of the editor then we will end
            // up in an inconsistent state between us and the editor where we have new tags but the
            // editor will never know.

            _tagSource.RegisterNotification(() =>
            {
                foreach (var change in changes)
                {
                    if (change.Key != _subjectBuffer)
                    {
                        continue;
                    }

                    // Now report them back to the UI on the main thread.
                    _batchChangeNotifier.EnqueueChanges(change.Value);
                }

            }, TaggerDelay.NearImmediate.ComputeTimeDelayMS(), CancellationToken.None);
        }

        public IEnumerable<ITagSpan<TTag>> GetTags(NormalizedSnapshotSpanCollection requestedSpans)
        {
            return GetTagsWorker(requestedSpans, accurate: false, cancellationToken: CancellationToken.None);
        }

        public IEnumerable<ITagSpan<TTag>> GetAllTags(NormalizedSnapshotSpanCollection requestedSpans, CancellationToken cancellationToken)
        {
            return GetTagsWorker(requestedSpans, accurate: true, cancellationToken: cancellationToken);
        }

        private IEnumerable<ITagSpan<TTag>> GetTagsWorker(
            NormalizedSnapshotSpanCollection requestedSpans,
            bool accurate,
            CancellationToken cancellationToken)
        {
            if (requestedSpans.Count == 0)
            {
                return SpecializedCollections.EmptyEnumerable<ITagSpan<TTag>>();
            }

            var buffer = requestedSpans.First().Snapshot.TextBuffer;
            var tags = accurate
                ? _tagSource.GetAccurateTagIntervalTreeForBuffer(buffer, cancellationToken)
                : _tagSource.GetTagIntervalTreeForBuffer(buffer);

            if (tags == null)
            {
                return SpecializedCollections.EmptyEnumerable<ITagSpan<TTag>>();
            }

            var result = GetTags(requestedSpans, tags);

            DebugVerifyTags(requestedSpans, result);

            return result;
        }

        private static IEnumerable<ITagSpan<TTag>> GetTags(NormalizedSnapshotSpanCollection requestedSpans, ITagSpanIntervalTree<TTag> tags)
        {
            // Special case the case where there is only one requested span.  In that case, we don't
            // need to allocate any intermediate collections
            return requestedSpans.Count == 1
                ? tags.GetIntersectingSpans(requestedSpans[0])
                : requestedSpans.Count < MaxNumberOfRequestedSpans
                    ? GetTagsForSmallNumberOfSpans(requestedSpans, tags)
                    : GetTagsForLargeNumberOfSpans(requestedSpans, tags);
        }

        private static IEnumerable<ITagSpan<TTag>> GetTagsForSmallNumberOfSpans(
            NormalizedSnapshotSpanCollection requestedSpans,
            ITagSpanIntervalTree<TTag> tags)
        {
            var result = new List<ITagSpan<TTag>>();

            foreach (var s in requestedSpans)
{'fraction_non_alphanumeric': 0.06698747326611672, 'fraction_numerical': 0.0012221203788573174, 'mean_word_length': 2.228057199211045, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5449828', 'n_tokens_mistral': 3231, 'n_tokens_neox': 3105, 'n_words': 843}