<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import * as vscode from 'vscode';
import * as jsoncParser from 'jsonc-parser';

export function activate(context: vscode.ExtensionContext): any {

	const tokenModifiers = ['static', 'abstract', 'deprecated', 'declaration', 'documentation', 'member', 'async'];
	const tokenTypes = ['types', 'structs', 'classes', 'interfaces', 'enums', 'parameterTypes', 'functions', 'variables'];

	const legend = new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);

	const semanticHighlightProvider: vscode.SemanticTokensProvider = {
		provideSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
			const builder = new vscode.SemanticTokensBuilder();

			function addToken(type: string, modifiers: string[], startLine: number, startCharacter: number, length: number) {
				let tokenType = legend.tokenTypes.indexOf(type);
				if (tokenType === -1) {
					tokenType = 0;
				}

				let tokenModifiers = 0;
				for (let i = 0; i < modifiers.length; i++) {
					const index = legend.tokenModifiers.indexOf(modifiers[i]);
					if (index !== -1) {
						tokenModifiers = tokenModifiers | 1 << index;
					}
				}

				builder.push(startLine, startCharacter, length, tokenType, tokenModifiers);
			}

			const visitor: jsoncParser.JSONVisitor = {
				onObjectProperty: (property: string, _offset: number, length: number, startLine: number, startCharacter: number) => {
					const [type, ...modifiers] = property.split('.');
					addToken(type, modifiers, startLine, startCharacter, length);
				},
				onLiteralValue: (value: any, _offset: number, length: number, startLine: number, startCharacter: number) => {
					if (typeof value === 'string') {
						const [type, ...modifiers] = value.split('.');
						addToken(type, modifiers, startLine, startCharacter, length);
					}
				}
			};
			jsoncParser.visit(document.getText(), visitor);<commit_msg>semantic token test: fix length, skip unknown tokens<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import * as vscode from 'vscode';
import * as jsoncParser from 'jsonc-parser';

export function activate(context: vscode.ExtensionContext): any {

	const tokenTypes = ['types', 'structs', 'classes', 'interfaces', 'enums', 'parameterTypes', 'functions', 'variables'];
	const tokenModifiers = ['static', 'abstract', 'deprecated', 'declaration', 'documentation', 'member', 'async'];

	const legend = new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);

	const semanticHighlightProvider: vscode.SemanticTokensProvider = {
		provideSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
			const builder = new vscode.SemanticTokensBuilder();

			function addToken(value: string, startLine: number, startCharacter: number, length: number) {
				const [type, ...modifiers] = value.split('.');

				let tokenType = legend.tokenTypes.indexOf(type);
				if (tokenType === -1) {
					return;
				}

				let tokenModifiers = 0;
				for (let i = 0; i < modifiers.length; i++) {
					const index = legend.tokenModifiers.indexOf(modifiers[i]);
					if (index !== -1) {
						tokenModifiers = tokenModifiers | 1 << index;
					}
				}


				builder.push(startLine, startCharacter, length, tokenType, tokenModifiers);
			}

			const visitor: jsoncParser.JSONVisitor = {
				onObjectProperty: (property: string, _offset: number, _length: number, startLine: number, startCharacter: number) => {
					addToken(property, startLine, startCharacter, property.length + 2);
				},
				onLiteralValue: (value: any, _offset: number, length: number, startLine: number, startCharacter: number) => {
					if (typeof value === 'string') {
						addToken(value, startLine, startCharacter, length);
					}
				}
			};
			jsoncParser.visit(document.getText(), visitor);
{'fraction_non_alphanumeric': 0.2047191011235955, 'fraction_numerical': 0.002696629213483146, 'mean_word_length': 2.938938053097345, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6957782', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1188, 'n_words': 326}