<commit_before>﻿using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Linq;
using System.Runtime.InteropServices;
using Microsoft.VisualStudio;
using Microsoft.VisualStudio.Editor;
using Microsoft.VisualStudio.Language.Intellisense;
using Microsoft.VisualStudio.OLE.Interop;
using Microsoft.VisualStudio.Shell;
using Microsoft.VisualStudio.Text;
using Microsoft.VisualStudio.Text.Editor;
using Microsoft.VisualStudio.Text.Operations;
using Microsoft.VisualStudio.TextManager.Interop;
using Microsoft.VisualStudio.Utilities;
using Typewriter.VisualStudio;
using VSCommand = Microsoft.VisualStudio.VSConstants.VSStd2KCmdID;

namespace Typewriter.TemplateEditor.Controllers
{
    [Export(typeof(ICompletionSourceProvider))]
    [ContentType("tst")]
    [Name("token completion1")]
    [Order(Before = "High")]
    internal class CompletionSourceProvider : ICompletionSourceProvider
    {
        [Import]
        internal ITextStructureNavigatorSelectorService NavigatorService { get; set; }

        [Import]
        internal IGlyphService GlyphService { get; set; }

        public ICompletionSource TryCreateCompletionSource(ITextBuffer textBuffer)
        {
            return new CompletionSource(this, textBuffer, GlyphService);
        }
    }

    internal class CompletionSource : ICompletionSource
    {
        private readonly CompletionSourceProvider sourceProvider;
        private readonly ITextBuffer buffer;
        private readonly IGlyphService glyphService;

        public CompletionSource(CompletionSourceProvider sourceProvider, ITextBuffer buffer, IGlyphService glyphService)
        {
            this.sourceProvider = sourceProvider;
            this.buffer = buffer;
            this.glyphService = glyphService;
        }

        public void AugmentCompletionSession(ICompletionSession session, IList<CompletionSet> completionSets)
        {
            var point = session.TextView.Caret.Position.BufferPosition;
            var line = point.GetContainingLine();

            if (line == null) return;

            var text = line.GetText();
            var index = point - line.Start;

            var start = index;

            if (index > 0)
            {
                for (var i = index; i > 0; i--)
                {
                    var current = text[i - 1];
                    if (current != '$' && current != '_' && char.IsLetterOrDigit(current) == false)
                        break;
                    start = i - 1;
                }
            }<commit_msg>force statement completion filters to start on $<commit_after>﻿using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Linq;
using System.Runtime.InteropServices;
using Microsoft.VisualStudio;
using Microsoft.VisualStudio.Editor;
using Microsoft.VisualStudio.Language.Intellisense;
using Microsoft.VisualStudio.OLE.Interop;
using Microsoft.VisualStudio.Shell;
using Microsoft.VisualStudio.Text;
using Microsoft.VisualStudio.Text.Editor;
using Microsoft.VisualStudio.Text.Operations;
using Microsoft.VisualStudio.TextManager.Interop;
using Microsoft.VisualStudio.Utilities;
using Typewriter.VisualStudio;
using VSCommand = Microsoft.VisualStudio.VSConstants.VSStd2KCmdID;

namespace Typewriter.TemplateEditor.Controllers
{
    [Export(typeof(ICompletionSourceProvider))]
    [ContentType("tst")]
    [Name("token completion1")]
    [Order(Before = "High")]
    internal class CompletionSourceProvider : ICompletionSourceProvider
    {
        [Import]
        internal ITextStructureNavigatorSelectorService NavigatorService { get; set; }

        [Import]
        internal IGlyphService GlyphService { get; set; }

        public ICompletionSource TryCreateCompletionSource(ITextBuffer textBuffer)
        {
            return new CompletionSource(this, textBuffer, GlyphService);
        }
    }

    internal class CompletionSource : ICompletionSource
    {
        private readonly CompletionSourceProvider sourceProvider;
        private readonly ITextBuffer buffer;
        private readonly IGlyphService glyphService;

        public CompletionSource(CompletionSourceProvider sourceProvider, ITextBuffer buffer, IGlyphService glyphService)
        {
            this.sourceProvider = sourceProvider;
            this.buffer = buffer;
            this.glyphService = glyphService;
        }

        public void AugmentCompletionSession(ICompletionSession session, IList<CompletionSet> completionSets)
        {
            var point = session.TextView.Caret.Position.BufferPosition;
            var line = point.GetContainingLine();

            if (line == null) return;

            var text = line.GetText();
            var index = point - line.Start;

            var start = index;

            if (index > 0)
            {
                for (var i = index; i > 0; i--)
                {
                    var current = text[i - 1];
                    if (current == '$')
                    {
                        start = i - 1;
                        break;
                    }

                    if (current != '_' && char.IsLetterOrDigit(current) == false) break;

                    start = i - 1;
                }
            }
{'fraction_non_alphanumeric': 0.07961238837165115, 'fraction_numerical': 0.002470074102223067, 'mean_word_length': 2.5163660654642617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6270572', 'n_tokens_mistral': 1346, 'n_tokens_neox': 1276, 'n_words': 304}