<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
'use strict';

/**
 * Create a syntax highighter with a fully declarative JSON style lexer description
 * using regular expressions.
 */

import { IDisposable } from 'vs/base/common/lifecycle';
import * as modes from 'vs/editor/common/modes';
import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon';
import { IModeService } from 'vs/editor/common/services/modeService';
import { Token } from 'vs/editor/common/core/token';
import { NullState, nullTokenize, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';

export class MonarchStateStackElement {

	public readonly parent: MonarchStateStackElement;
	public readonly state: string;
	public readonly depth: number;

	constructor(parent: MonarchStateStackElement, state: string) {
		this.parent = parent;
		this.state = state;
		this.depth = (this.parent ? this.parent.depth : 0) + 1;
	}

	private static _equals(a: MonarchStateStackElement, b: MonarchStateStackElement): boolean {
		while (a !== null && b !== null) {
			if (a === b) {
				return true;
			}
			if (a.state !== b.state) {
				return false;
			}
			a = a.parent;
			b = b.parent;
		}
		if (a === null && b === null) {
			return true;
		}
		return false;
	}

	public equals(other: MonarchStateStackElement): boolean {
		return MonarchStateStackElement._equals(this, other);
	}

	public push(state: string): MonarchStateStackElement {
		return new MonarchStateStackElement(this, state);
	}

	public pop(): MonarchStateStackElement {
		return this.parent;
	}

	public print(): void {
		console.log(this._print());
		// console.log('PRINTING MY STACK');
		// console.log(this);
	}

	private _print(): string {
		if (this.parent) {
			return this.state + ' -> ' + this.parent._print();
		} else {
			return this.state + ' -|';
		}
	}

	public popall(): MonarchStateStackElement {
		let result: MonarchStateStackElement = this;
		while (result.parent) {
			result = result.parent;
		}
		return result;
	}

	public switchTo(state: string): MonarchStateStackElement {
		return new MonarchStateStackElement(this.parent, state);
	}
}

export class MonarchLineState implements modes.IState {

	private readonly _modeId: string;
	public readonly stack: MonarchStateStackElement;
	public readonly embeddedModeState: modes.IState;

	constructor(
		modeId: string,
		stack: MonarchStateStackElement,
		embeddedModeState: modes.IState
	) {
		if (!stack) {
			console.log('CREATING LINE STATE WITHOUT STACK!!!'); process.exit(0);
		}
		this._modeId = modeId;
		this.stack = stack;
		this.embeddedModeState = embeddedModeState;
	}

	public clone(): modes.IState {
		return this;
	}

	public equals(other: modes.IState): boolean {
		if (!(other instanceof MonarchLineState)) {
			return false;
		}
		if (this._modeId !== other._modeId) {
			return false;
		}
		return this.stack.equals(other.stack);
	}

	public getModeId(): string {
		return this._modeId;
	}
}

const hasOwnProperty = Object.hasOwnProperty;

export class MonarchTokenizer implements modes.ITokenizationSupport {

	private readonly _modeService: IModeService;
	private readonly _modeId: string;
	private readonly _lexer: monarchCommon.ILexer;
	private _embeddedModes: { [modeId: string]: boolean; };
	private _tokenizationRegistryListener: IDisposable;

	constructor(modeService: IModeService, modeId: string, lexer: monarchCommon.ILexer) {
		this._modeService = modeService;
		this._modeId = modeId;
		this._lexer = lexer;
		this._embeddedModes = Object.create(null);

		// Set up listening for embedded modes
		let emitting = false;
		this._tokenizationRegistryListener = modes.TokenizationRegistry.onDidChange((e) => {
			if (emitting) {
				return;
			}
			let isOneOfMyEmbeddedModes = this._embeddedModes[e.languageId];
			if (isOneOfMyEmbeddedModes) {
				emitting = true;
				modes.TokenizationRegistry.fire(this._modeId);
				emitting = false;
			}
		});
	}

	public dispose(): void {
		this._tokenizationRegistryListener.dispose();
	}

	public getInitialState(): modes.IState {
		let rootState = new MonarchStateStackElement(null, this._lexer.start);
		return new MonarchLineState(this._modeId, rootState, null);
	}

	public tokenize(line: string, _lineState: modes.IState, offsetDelta: number): modes.ILineTokens {
		let lineState = (<MonarchLineState>_lineState);
		if (lineState.embeddedModeState) {
			return this._nestedTokenize(line, lineState, offsetDelta, [], []);
		} else {
			return this._myTokenize(line, lineState, offsetDelta, [], []);
		}<commit_msg>Reuse the same instances of MonarchStackElement up to a certain depth<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
'use strict';

/**
 * Create a syntax highighter with a fully declarative JSON style lexer description
 * using regular expressions.
 */

import { IDisposable } from 'vs/base/common/lifecycle';
import * as modes from 'vs/editor/common/modes';
import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon';
import { IModeService } from 'vs/editor/common/services/modeService';
import { Token } from 'vs/editor/common/core/token';
import { NullState, nullTokenize, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';

/**
 * Reuse the same stack elements up to a certain depth.
 */
export class MonarchStackElementFactory {

	private static _INSTANCE = new MonarchStackElementFactory(5);
	public static create(parent: MonarchStackElement, state: string): MonarchStackElement {
		return this._INSTANCE.create(parent, state);
	}

	private readonly _maxCacheDepth: number;
	private readonly _entries: { [stackElementId: string]: MonarchStackElement; };

	constructor(maxCacheDepth: number) {
		this._maxCacheDepth = maxCacheDepth;
		this._entries = Object.create(null);
	}

	public create(parent: MonarchStackElement, state: string): MonarchStackElement {
		if (parent !== null && parent.depth >= this._maxCacheDepth) {
			return new MonarchStackElement(parent, state);
		}
		let stackElementId = MonarchStackElementFactory._getStackElementId(parent);
		if (stackElementId.length > 0) {
			stackElementId += '|';
		}
		stackElementId += state;

		let result = this._entries[stackElementId];
		if (result) {
			return result;
		}
		result = new MonarchStackElement(parent, state);
		this._entries[stackElementId] = result;
		return result;
	}

	private static _getStackElementId(element: MonarchStackElement): string {
		let result = '';
		while (element !== null) {
			if (result.length > 0) {
				result += '|';
			}
			result += element.state;
			element = element.parent;
		}
		return result;
	}
}

export class MonarchStackElement {

	public readonly parent: MonarchStackElement;
	public readonly state: string;
	public readonly depth: number;

	constructor(parent: MonarchStackElement, state: string) {
		this.parent = parent;
		this.state = state;
		this.depth = (this.parent ? this.parent.depth : 0) + 1;
	}

	private static _equals(a: MonarchStackElement, b: MonarchStackElement): boolean {
		while (a !== null && b !== null) {
			if (a === b) {
				return true;
			}
			if (a.state !== b.state) {
				return false;
			}
			a = a.parent;
			b = b.parent;
		}
		if (a === null && b === null) {
			return true;
		}
		return false;
	}

	public equals(other: MonarchStackElement): boolean {
		return MonarchStackElement._equals(this, other);
	}

	public push(state: string): MonarchStackElement {
		return MonarchStackElementFactory.create(this, state);
	}

	public pop(): MonarchStackElement {
		return this.parent;
	}

	public print(): void {
		console.log(this._print());
		// console.log('PRINTING MY STACK');
		// console.log(this);
	}

	private _print(): string {
		if (this.parent) {
			return this.state + ' -> ' + this.parent._print();
		} else {
			return this.state + ' -|';
		}
	}

	public popall(): MonarchStackElement {
		let result: MonarchStackElement = this;
		while (result.parent) {
			result = result.parent;
		}
		return result;
	}

	public switchTo(state: string): MonarchStackElement {
		return MonarchStackElementFactory.create(this.parent, state);
	}
}

export class MonarchLineState implements modes.IState {

	private readonly _modeId: string;
	public readonly stack: MonarchStackElement;
	public readonly embeddedModeState: modes.IState;

	constructor(
		modeId: string,
		stack: MonarchStackElement,
		embeddedModeState: modes.IState
	) {
		if (!stack) {
			console.log('CREATING LINE STATE WITHOUT STACK!!!'); process.exit(0);
		}
		this._modeId = modeId;
		this.stack = stack;
		this.embeddedModeState = embeddedModeState;
	}

	public clone(): modes.IState {
		return this;
	}

	public equals(other: modes.IState): boolean {
		if (!(other instanceof MonarchLineState)) {
			return false;
		}
		if (this._modeId !== other._modeId) {
			return false;
		}
		return this.stack.equals(other.stack);
	}

	public getModeId(): string {
		return this._modeId;
	}
}

const hasOwnProperty = Object.hasOwnProperty;

export class MonarchTokenizer implements modes.ITokenizationSupport {

	private readonly _modeService: IModeService;
	private readonly _modeId: string;
	private readonly _lexer: monarchCommon.ILexer;
	private _embeddedModes: { [modeId: string]: boolean; };
	private _tokenizationRegistryListener: IDisposable;

	constructor(modeService: IModeService, modeId: string, lexer: monarchCommon.ILexer) {
		this._modeService = modeService;
		this._modeId = modeId;
		this._lexer = lexer;
		this._embeddedModes = Object.create(null);

		// Set up listening for embedded modes
		let emitting = false;
		this._tokenizationRegistryListener = modes.TokenizationRegistry.onDidChange((e) => {
			if (emitting) {
				return;
			}
			let isOneOfMyEmbeddedModes = this._embeddedModes[e.languageId];
			if (isOneOfMyEmbeddedModes) {
				emitting = true;
				modes.TokenizationRegistry.fire(this._modeId);
				emitting = false;
			}
		});
	}

	public dispose(): void {
		this._tokenizationRegistryListener.dispose();
	}

	public getInitialState(): modes.IState {
		let rootState = MonarchStackElementFactory.create(null, this._lexer.start);
		return new MonarchLineState(this._modeId, rootState, null);
	}

	public tokenize(line: string, _lineState: modes.IState, offsetDelta: number): modes.ILineTokens {
		let lineState = (<MonarchLineState>_lineState);
		if (lineState.embeddedModeState) {
			return this._nestedTokenize(line, lineState, offsetDelta, [], []);
		} else {
			return this._myTokenize(line, lineState, offsetDelta, [], []);
		}
{'fraction_non_alphanumeric': 0.1459227467811159, 'fraction_numerical': 0.0008047210300429185, 'mean_word_length': 3.6623593163818255, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3012638', 'n_tokens_mistral': 3690, 'n_tokens_neox': 3330, 'n_words': 838}