<commit_before>﻿using Microsoft.VisualStudio.Text;
using Papyrus.Common;
using Papyrus.Common.Extensions;
using Papyrus.Language.Components;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Papyrus.Language {
    public class IdentifierTokenScanner : TokenScannerModule {
        public override bool Scan(SnapshotSpan sourceSnapshotSpan, int offset, ref TokenScannerState state, out Token token) {
            string text = sourceSnapshotSpan.GetText();
            int length = Delimiter.FindNext(text, offset) - offset;
            if (Identifier.IsValid(text.Substring(offset, length))) {
                token = new Token(new Identifier(text.Substring(offset, length)), sourceSnapshotSpan.Subspan(offset, length));
                return true;
            }

            token = null;
            return false;
        }
    }
}<commit_msg>Update IdentifierTokenScanner.cs<commit_after>﻿using Microsoft.VisualStudio.Text;
using Papyrus.Common;
using Papyrus.Common.Extensions;
using Papyrus.Language.Components;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Papyrus.Language {
    public class IdentifierTokenScanner : TokenScannerModule {
        public override bool Scan(SnapshotSpan sourceSnapshotSpan, int offset, ref TokenScannerState state, out Token token) {
            if (state == TokenScannerState.Text) {
                string text = sourceSnapshotSpan.GetText();
                int length = Delimiter.FindNext(text, offset) - offset;
                if (Identifier.IsValid(text.Substring(offset, length))) {
                    token = new Token(new Identifier(text.Substring(offset, length)), sourceSnapshotSpan.Subspan(offset, length));
                    return true;
                }
            }

            token = null;
            return false;
        }
    }
}
{'fraction_non_alphanumeric': 0.08700102354145343, 'fraction_numerical': 0.0, 'mean_word_length': 2.8484251968503935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2649937', 'n_tokens_mistral': 511, 'n_tokens_neox': 493, 'n_words': 126}