<commit_before>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Threading;
using Microsoft.CodeAnalysis.Classification.Classifiers;
using Microsoft.CodeAnalysis.Text;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.Classification
{
    internal partial class AbstractClassificationService
    {
        private struct Worker
        {
            private readonly SemanticModel _semanticModel;
            private readonly SyntaxTree _syntaxTree;
            private readonly TextSpan _textSpan;
            private readonly List<ClassifiedSpan> _list;
            private readonly CancellationToken _cancellationToken;
            private readonly Func<SyntaxNode, List<ISyntaxClassifier>> _getNodeClassifiers;
            private readonly Func<SyntaxToken, List<ISyntaxClassifier>> _getTokenClassifiers;
            private readonly HashSet<ClassifiedSpan> _set;
            private readonly Stack<SyntaxNodeOrToken> _pendingNodes;

            private Worker(
                Workspace workspace,
                SemanticModel semanticModel,
                TextSpan textSpan,
                List<ClassifiedSpan> list,
                Func<SyntaxNode, List<ISyntaxClassifier>> getNodeClassifiers,
                Func<SyntaxToken, List<ISyntaxClassifier>> getTokenClassifiers,
                CancellationToken cancellationToken)
            {
                _getNodeClassifiers = getNodeClassifiers;
                _getTokenClassifiers = getTokenClassifiers;
                _semanticModel = semanticModel;
                _syntaxTree = semanticModel.SyntaxTree;
                _textSpan = textSpan;
                _list = list;
                _cancellationToken = cancellationToken;

                // get one from pool
                _set = SharedPools.Default<HashSet<ClassifiedSpan>>().AllocateAndClear();
                _pendingNodes = SharedPools.Default<Stack<SyntaxNodeOrToken>>().AllocateAndClear();
            }

            internal static void Classify(
                Workspace workspace,
                SemanticModel semanticModel,
                TextSpan textSpan,
                List<ClassifiedSpan> list,
                Func<SyntaxNode, List<ISyntaxClassifier>> getNodeClassifiers,
                Func<SyntaxToken, List<ISyntaxClassifier>> getTokenClassifiers,
                CancellationToken cancellationToken)
            {
                var worker = new Worker(workspace, semanticModel, textSpan, list, getNodeClassifiers, getTokenClassifiers, cancellationToken);

                try
                {
                    worker._pendingNodes.Push(worker._syntaxTree.GetRoot(cancellationToken));
                    worker.ProcessNodes();
                }
                finally
                {
                    // release collections to the pool
                    SharedPools.Default<HashSet<ClassifiedSpan>>().ClearAndFree(worker._set);
                    SharedPools.Default<Stack<SyntaxNodeOrToken>>().ClearAndFree(worker._pendingNodes);
                }
            }

            private void AddClassification(TextSpan textSpan, string type)
            {
                var tuple = new ClassifiedSpan(type, textSpan);
                if (!_set.Contains(tuple))
                {
                    _list.Add(tuple);
                    _set.Add(tuple);
                }
            }

            private void ProcessNodes()
            {
                while (_pendingNodes.Count > 0)
                {
                    _cancellationToken.ThrowIfCancellationRequested();
                    var nodeOrToken = _pendingNodes.Pop();

                    if (nodeOrToken.Span.IntersectsWith(_textSpan))
                    {
                        ClassifyNodeOrToken(nodeOrToken);

                        foreach (var child in nodeOrToken.ChildNodesAndTokens())
                        {
                            _pendingNodes.Push(child);
                        }
                    }
                }<commit_msg>Do intersection with fullspans.  Otherwise we will not classify doc comments on a node when only asking to classify a subspan of a document.<commit_after>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Threading;
using Microsoft.CodeAnalysis.Classification.Classifiers;
using Microsoft.CodeAnalysis.Text;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.Classification
{
    internal partial class AbstractClassificationService
    {
        private struct Worker
        {
            private readonly SemanticModel _semanticModel;
            private readonly SyntaxTree _syntaxTree;
            private readonly TextSpan _textSpan;
            private readonly List<ClassifiedSpan> _list;
            private readonly CancellationToken _cancellationToken;
            private readonly Func<SyntaxNode, List<ISyntaxClassifier>> _getNodeClassifiers;
            private readonly Func<SyntaxToken, List<ISyntaxClassifier>> _getTokenClassifiers;
            private readonly HashSet<ClassifiedSpan> _set;
            private readonly Stack<SyntaxNodeOrToken> _pendingNodes;

            private Worker(
                Workspace workspace,
                SemanticModel semanticModel,
                TextSpan textSpan,
                List<ClassifiedSpan> list,
                Func<SyntaxNode, List<ISyntaxClassifier>> getNodeClassifiers,
                Func<SyntaxToken, List<ISyntaxClassifier>> getTokenClassifiers,
                CancellationToken cancellationToken)
            {
                _getNodeClassifiers = getNodeClassifiers;
                _getTokenClassifiers = getTokenClassifiers;
                _semanticModel = semanticModel;
                _syntaxTree = semanticModel.SyntaxTree;
                _textSpan = textSpan;
                _list = list;
                _cancellationToken = cancellationToken;

                // get one from pool
                _set = SharedPools.Default<HashSet<ClassifiedSpan>>().AllocateAndClear();
                _pendingNodes = SharedPools.Default<Stack<SyntaxNodeOrToken>>().AllocateAndClear();
            }

            internal static void Classify(
                Workspace workspace,
                SemanticModel semanticModel,
                TextSpan textSpan,
                List<ClassifiedSpan> list,
                Func<SyntaxNode, List<ISyntaxClassifier>> getNodeClassifiers,
                Func<SyntaxToken, List<ISyntaxClassifier>> getTokenClassifiers,
                CancellationToken cancellationToken)
            {
                var worker = new Worker(workspace, semanticModel, textSpan, list, getNodeClassifiers, getTokenClassifiers, cancellationToken);

                try
                {
                    worker._pendingNodes.Push(worker._syntaxTree.GetRoot(cancellationToken));
                    worker.ProcessNodes();
                }
                finally
                {
                    // release collections to the pool
                    SharedPools.Default<HashSet<ClassifiedSpan>>().ClearAndFree(worker._set);
                    SharedPools.Default<Stack<SyntaxNodeOrToken>>().ClearAndFree(worker._pendingNodes);
                }
            }

            private void AddClassification(TextSpan textSpan, string type)
            {
                var tuple = new ClassifiedSpan(type, textSpan);
                if (!_set.Contains(tuple))
                {
                    _list.Add(tuple);
                    _set.Add(tuple);
                }
            }

            private void ProcessNodes()
            {
                while (_pendingNodes.Count > 0)
                {
                    _cancellationToken.ThrowIfCancellationRequested();
                    var nodeOrToken = _pendingNodes.Pop();

                    if (nodeOrToken.FullSpan.IntersectsWith(_textSpan))
                    {
                        ClassifyNodeOrToken(nodeOrToken);

                        foreach (var child in nodeOrToken.ChildNodesAndTokens())
                        {
                            _pendingNodes.Push(child);
                        }
                    }
                }
{'fraction_non_alphanumeric': 0.06247789697041141, 'fraction_numerical': 0.0007072969468348461, 'mean_word_length': 1.7465199093557786, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '21037', 'n_tokens_mistral': 1988, 'n_tokens_neox': 1866, 'n_words': 417}