<commit_before>/* --------------------------------------------------------------------------------------------
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See License.txt in the project root for license information.
 * ------------------------------------------------------------------------------------------ */
'use strict';

import { TextDocumentIdentifier, Range } from 'vscode-languageserver-types';
import { PartialResultParams, WorkDoneProgressParams, WorkDoneProgressOptions, TextDocumentRegistrationOptions, StaticRegistrationOptions } from './protocol';
import { ProtocolRequestType } from './messages';

/**
 * A set of predefined token types. This set is not fixed
 * an clients can specify additional token types via the
 * corresponding client capabilities.
 *
 * @since 3.16.0 - Proposed state
 */
export enum SemanticTokenTypes {
	comment = 'comment',
	keyword = 'keyword',
	string = 'string',
	number = 'number',
	regexp = 'regexp',
	operator = 'operator',
	namespace = 'namespace',
	type = 'type',
	struct = 'struct',
	class = 'class',
	interface = 'interface',
	enum = 'enum',
	typeParameter = 'typeParameter',
	function = 'function',
	member = 'member',
	property = 'property',
	macro = 'macro',
	variable = 'variable',
	parameter = 'parameter',
	label = 'label'
}

/**
 * A set of predefined token modifiers. This set is not fixed
 * an clients can specify additional token types via the
 * corresponding client capabilities.
 *
 * @since 3.16.0 - Proposed state
 */
export enum SemanticTokenModifiers {
	documentation = 'documentation',
	declaration = 'declaration',
	definition = 'definition',
	reference = 'reference',
	static = 'static',
	abstract = 'abstract',
	deprecated = 'deprecated',
	async = 'async',
	volatile = 'volatile',
	readonly = 'readonly'
}

/**<commit_msg>Remove references token modifier.<commit_after>/* --------------------------------------------------------------------------------------------
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License. See License.txt in the project root for license information.
 * ------------------------------------------------------------------------------------------ */
'use strict';

import { TextDocumentIdentifier, Range } from 'vscode-languageserver-types';
import { PartialResultParams, WorkDoneProgressParams, WorkDoneProgressOptions, TextDocumentRegistrationOptions, StaticRegistrationOptions } from './protocol';
import { ProtocolRequestType } from './messages';

/**
 * A set of predefined token types. This set is not fixed
 * an clients can specify additional token types via the
 * corresponding client capabilities.
 *
 * @since 3.16.0 - Proposed state
 */
export enum SemanticTokenTypes {
	comment = 'comment',
	keyword = 'keyword',
	string = 'string',
	number = 'number',
	regexp = 'regexp',
	operator = 'operator',
	namespace = 'namespace',
	type = 'type',
	struct = 'struct',
	class = 'class',
	interface = 'interface',
	enum = 'enum',
	typeParameter = 'typeParameter',
	function = 'function',
	member = 'member',
	property = 'property',
	macro = 'macro',
	variable = 'variable',
	parameter = 'parameter',
	label = 'label'
}

/**
 * A set of predefined token modifiers. This set is not fixed
 * an clients can specify additional token types via the
 * corresponding client capabilities.
 *
 * @since 3.16.0 - Proposed state
 */
export enum SemanticTokenModifiers {
	documentation = 'documentation',
	declaration = 'declaration',
	definition = 'definition',
	static = 'static',
	abstract = 'abstract',
	deprecated = 'deprecated',
	async = 'async',
	volatile = 'volatile',
	readonly = 'readonly'
}

/**
{'fraction_non_alphanumeric': 0.20763400108283703, 'fraction_numerical': 0.004331348132106118, 'mean_word_length': 2.8290155440414506, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6606084', 'n_tokens_mistral': 946, 'n_tokens_neox': 916, 'n_words': 323}