<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
'use strict';

import { IDiffChange, ISequence, LcsDiff } from 'vs/base/common/diff/diff';
import * as strings from 'vs/base/common/strings';
import { ICharChange, ILineChange } from 'vs/editor/common/editorCommon';

const MAXIMUM_RUN_TIME = 5000; // 5 seconds
const MINIMUM_MATCHING_CHARACTER_LENGTH = 3;

interface IMarker {
	lineNumber: number;
	column: number;
	offset: number;
}

function computeDiff(originalSequence: ISequence, modifiedSequence: ISequence, continueProcessingPredicate: () => boolean, pretty: boolean): IDiffChange[] {
	const diffAlgo = new LcsDiff(originalSequence, modifiedSequence, continueProcessingPredicate);
	return diffAlgo.ComputeDiff(pretty);
}

class MarkerSequence implements ISequence {

	public buffer: string;
	public startMarkers: IMarker[];
	public endMarkers: IMarker[];

	constructor(buffer: string, startMarkers: IMarker[], endMarkers: IMarker[]) {
		this.buffer = buffer;
		this.startMarkers = startMarkers;
		this.endMarkers = endMarkers;
	}

	public getLength(): number {
		return this.startMarkers.length;
	}

	public getElementHash(i: number): string {
		return this.buffer.substring(this.startMarkers[i].offset, this.endMarkers[i].offset);
	}

	public getStartLineNumber(i: number): number {
		if (i === this.startMarkers.length) {
			// This is the special case where a change happened after the last marker
			return this.startMarkers[i - 1].lineNumber + 1;
		}
		return this.startMarkers[i].lineNumber;
	}

	public getStartColumn(i: number): number {
		return this.startMarkers[i].column;
	}

	public getEndLineNumber(i: number): number {
		return this.endMarkers[i].lineNumber;
	}

	public getEndColumn(i: number): number {
		return this.endMarkers[i].column;
	}

}

class LineMarkerSequence extends MarkerSequence {

	constructor(lines: string[]) {
		let buffer = '';
		let startMarkers: IMarker[] = [];
		let endMarkers: IMarker[] = [];

		for (let pos = 0, i = 0, length = lines.length; i < length; i++) {
			buffer += lines[i];
			const startColumn = LineMarkerSequence._getFirstNonBlankColumn(lines[i], 1);
			const endColumn = LineMarkerSequence._getLastNonBlankColumn(lines[i], 1);

			startMarkers.push({
				offset: pos + startColumn - 1,
				lineNumber: i + 1,
				column: startColumn
			});

			endMarkers.push({
				offset: pos + endColumn - 1,
				lineNumber: i + 1,
				column: endColumn
			});

			pos += lines[i].length;
		}

		super(buffer, startMarkers, endMarkers);
	}

	public static _getFirstNonBlankColumn(txt: string, defaultValue: number): number {
		const r = strings.firstNonWhitespaceIndex(txt);
		if (r === -1) {
			return defaultValue;
		}
		return r + 1;
	}

	public static _getLastNonBlankColumn(txt: string, defaultValue: number): number {
		const r = strings.lastNonWhitespaceIndex(txt);
		if (r === -1) {
			return defaultValue;
		}
		return r + 2;
	}

	public getCharSequence(startIndex: number, endIndex: number): MarkerSequence {
		let startMarkers: IMarker[] = [];
		let endMarkers: IMarker[] = [];
		for (let index = startIndex; index <= endIndex; index++) {
			const startMarker = this.startMarkers[index];
			const endMarker = this.endMarkers[index];
			for (let i = startMarker.offset; i < endMarker.offset; i++) {
				startMarkers.push({
					offset: i,
					lineNumber: startMarker.lineNumber,
					column: startMarker.column + (i - startMarker.offset)
				});
				endMarkers.push({
					offset: i + 1,
					lineNumber: startMarker.lineNumber,
					column: startMarker.column + (i - startMarker.offset) + 1
				});
			}
		}
		return new MarkerSequence(this.buffer, startMarkers, endMarkers);
	}
}

class CharChange implements ICharChange {

	public originalStartLineNumber: number;
	public originalStartColumn: number;
	public originalEndLineNumber: number;
	public originalEndColumn: number;

	public modifiedStartLineNumber: number;
	public modifiedStartColumn: number;
	public modifiedEndLineNumber: number;
	public modifiedEndColumn: number;

	constructor(
		originalStartLineNumber: number,
		originalStartColumn: number,
		originalEndLineNumber: number,
		originalEndColumn: number,
		modifiedStartLineNumber: number,
		modifiedStartColumn: number,
		modifiedEndLineNumber: number,
		modifiedEndColumn: number
	) {
		this.originalStartLineNumber = originalStartLineNumber;
		this.originalStartColumn = originalStartColumn;
		this.originalEndLineNumber = originalEndLineNumber;
		this.originalEndColumn = originalEndColumn;
		this.modifiedStartLineNumber = modifiedStartLineNumber;
		this.modifiedStartColumn = modifiedStartColumn;
		this.modifiedEndLineNumber = modifiedEndLineNumber;
		this.modifiedEndColumn = modifiedEndColumn;
	}

	public static createFromDiffChange(diffChange: IDiffChange, originalCharSequence: MarkerSequence, modifiedCharSequence: MarkerSequence): CharChange {
		let originalStartLineNumber: number;
		let originalStartColumn: number;
		let originalEndLineNumber: number;
		let originalEndColumn: number;
		let modifiedStartLineNumber: number;
		let modifiedStartColumn: number;
		let modifiedEndLineNumber: number;
		let modifiedEndColumn: number;

		if (diffChange.originalLength === 0) {
			originalStartLineNumber = 0;
			originalStartColumn = 0;
			originalEndLineNumber = 0;
			originalEndColumn = 0;
		} else {
			originalStartLineNumber = originalCharSequence.getStartLineNumber(diffChange.originalStart);<commit_msg>Improve diff computer<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
'use strict';

import { IDiffChange, ISequence, LcsDiff } from 'vs/base/common/diff/diff';
import * as strings from 'vs/base/common/strings';
import { ICharChange, ILineChange } from 'vs/editor/common/editorCommon';

const MAXIMUM_RUN_TIME = 5000; // 5 seconds
const MINIMUM_MATCHING_CHARACTER_LENGTH = 3;

function computeDiff(originalSequence: ISequence, modifiedSequence: ISequence, continueProcessingPredicate: () => boolean, pretty: boolean): IDiffChange[] {
	const diffAlgo = new LcsDiff(originalSequence, modifiedSequence, continueProcessingPredicate);
	return diffAlgo.ComputeDiff(pretty);
}

class LineMarkerSequence implements ISequence {

	private readonly _lines: string[];
	private readonly _startColumns: number[];
	private readonly _endColumns: number[];

	constructor(lines: string[]) {
		let startColumns: number[] = [];
		let endColumns: number[] = [];
		for (let i = 0, length = lines.length; i < length; i++) {
			startColumns[i] = LineMarkerSequence._getFirstNonBlankColumn(lines[i], 1);
			endColumns[i] = LineMarkerSequence._getLastNonBlankColumn(lines[i], 1);
		}
		this._lines = lines;
		this._startColumns = startColumns;
		this._endColumns = endColumns;
	}

	public getLength(): number {
		return this._lines.length;
	}

	public getElementHash(i: number): string {
		return this._lines[i].substring(this._startColumns[i] - 1, this._endColumns[i] - 1);
	}

	public getStartLineNumber(i: number): number {
		return i + 1;
	}

	public getStartColumn(i: number): number {
		return this._startColumns[i];
	}

	public getEndLineNumber(i: number): number {
		return i + 1;
	}

	public getEndColumn(i: number): number {
		return this._endColumns[i];
	}

	public static _getFirstNonBlankColumn(txt: string, defaultValue: number): number {
		const r = strings.firstNonWhitespaceIndex(txt);
		if (r === -1) {
			return defaultValue;
		}
		return r + 1;
	}

	public static _getLastNonBlankColumn(txt: string, defaultValue: number): number {
		const r = strings.lastNonWhitespaceIndex(txt);
		if (r === -1) {
			return defaultValue;
		}
		return r + 2;
	}

	public getCharSequence(startIndex: number, endIndex: number): CharSequence {
		let chars: string[] = [];
		let lineNumbers: number[] = [];
		let columns: number[] = [];
		let len = 0;
		for (let index = startIndex; index <= endIndex; index++) {
			const startColumn = this._startColumns[index];
			const endColumn = this._endColumns[index];
			for (let col = startColumn; col < endColumn; col++) {
				chars[len] = this._lines[index].charAt(col - 1);
				lineNumbers[len] = index + 1;
				columns[len] = col;
				len++;
			}
		}
		return new CharSequence(chars, lineNumbers, columns);
	}
}

class CharSequence implements ISequence {

	private readonly _chars: string[];
	private readonly _lineNumbers: number[];
	private readonly _columns: number[];

	constructor(chars: string[], lineNumbers: number[], columns: number[]) {
		this._chars = chars;
		this._lineNumbers = lineNumbers;
		this._columns = columns;
	}

	public getLength(): number {
		return this._chars.length;
	}

	public getElementHash(i: number): string {
		return this._chars[i];
	}

	public getStartLineNumber(i: number): number {
		return this._lineNumbers[i];
	}

	public getStartColumn(i: number): number {
		return this._columns[i];
	}

	public getEndLineNumber(i: number): number {
		return this._lineNumbers[i];
	}

	public getEndColumn(i: number): number {
		return this._columns[i] + 1;
	}
}

class CharChange implements ICharChange {

	public originalStartLineNumber: number;
	public originalStartColumn: number;
	public originalEndLineNumber: number;
	public originalEndColumn: number;

	public modifiedStartLineNumber: number;
	public modifiedStartColumn: number;
	public modifiedEndLineNumber: number;
	public modifiedEndColumn: number;

	constructor(
		originalStartLineNumber: number,
		originalStartColumn: number,
		originalEndLineNumber: number,
		originalEndColumn: number,
		modifiedStartLineNumber: number,
		modifiedStartColumn: number,
		modifiedEndLineNumber: number,
		modifiedEndColumn: number
	) {
		this.originalStartLineNumber = originalStartLineNumber;
		this.originalStartColumn = originalStartColumn;
		this.originalEndLineNumber = originalEndLineNumber;
		this.originalEndColumn = originalEndColumn;
		this.modifiedStartLineNumber = modifiedStartLineNumber;
		this.modifiedStartColumn = modifiedStartColumn;
		this.modifiedEndLineNumber = modifiedEndLineNumber;
		this.modifiedEndColumn = modifiedEndColumn;
	}

	public static createFromDiffChange(diffChange: IDiffChange, originalCharSequence: CharSequence, modifiedCharSequence: CharSequence): CharChange {
		let originalStartLineNumber: number;
		let originalStartColumn: number;
		let originalEndLineNumber: number;
		let originalEndColumn: number;
		let modifiedStartLineNumber: number;
		let modifiedStartColumn: number;
		let modifiedEndLineNumber: number;
		let modifiedEndColumn: number;

		if (diffChange.originalLength === 0) {
			originalStartLineNumber = 0;
			originalStartColumn = 0;
			originalEndLineNumber = 0;
			originalEndColumn = 0;
		} else {
			originalStartLineNumber = originalCharSequence.getStartLineNumber(diffChange.originalStart);
{'fraction_non_alphanumeric': 0.1334927805828683, 'fraction_numerical': 0.004694835680751174, 'mean_word_length': 4.129486597001363, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '964856', 'n_tokens_mistral': 3603, 'n_tokens_neox': 3253, 'n_words': 828}