<commit_before>/**
 * Created by danek_000 on 30.8.2015.
 *
 *   .::::::::..          ..::::::::.
 * 	:::::::::::::        :::::::::::::
 * :::::::::::' .\      /. `:::::::::::
 * `::::::::::_,__o    o__,_::::::::::'
 */
define(
	[
		'./Record',
		'./util/arrayUtils'
	],
	function(
		Record
	) {
		'use strict';

		/**
		 * Describes a traverser to execute a DFA suggesting input expansion if required.
		 *
		 * @param biverseDFA
		 * @constructor
		 */
		function Traverser (biverseDFA) {

			// Define an initial state
			this.initialState = biverseDFA.initialState;

			// Define the conventional transitions table
			this.transitions = biverseDFA.transitions;

			// Define the transported transitions table
			this.transportedTransitions = transportTransitions(biverseDFA.transitions);

			// Define the final states
			this.finalStates = biverseDFA.finalStates;

			// Define the input buffer
			this.inputBuffer = [];

			// Define the indicator of input completeness
			this.inputOver = false;

			// Define the final records
			this.finalRecords = [];
		}

		/**
		 * Create transported transition table out of a given transition table.
		 *
		 * @param transitions
		 * @returns {Array}
		 */
		function transportTransitions (transitions) {

			var transportedTransitions = [];

			var statesCount = transitions.length;

			for (var stateNumber = 0; stateNumber < statesCount; stateNumber ++) {
				transportedTransitions[stateNumber] = {};

				var stateTransitionKeys = Object.keys(transitions[stateNumber]);

				var stateTransitionKeysCount = stateTransitionKeys.length;

				for (var stateTransitionKeyId = 0; stateTransitionKeyId < stateTransitionKeysCount; stateTransitionKeyId ++) {
					var stateTransitionKey = stateTransitionKeys[stateTransitionKeyId];

					var transition = transitions[stateNumber][stateTransitionKey];

					var transitionString = transition + '';

					if (transportedTransitions[stateNumber][transitionString] === undefined) {
						transportedTransitions[stateNumber][transitionString] = [];
					}

					transportedTransitions[stateNumber][transitionString].push(stateTransitionKey);
				}
			}

			return transportedTransitions;
		}

		/**
		 * Get and buffer the next input item
		 *
		 * @param traverser
		 * @param input
		 */
		function saveNextInputItem (traverser, input) {

			// If input is not yet over
			if (!isInputOver(traverser)) {

				// Get the first input item
				var inputItem = input();

				// If next input exists
				if (inputItem !== null) {

					// Save input item to the buffer
					traverser.inputBuffer.push(inputItem);
				} else {

					// State that input is over
					traverser.inputOver = true;
				}
			}
		}

		/**
		 * Checks whether or not the input is over.
		 *
		 * @param traverser
		 * @returns {boolean|*}
		 */
		function isInputOver (traverser) {
			return traverser.inputOver;
		}

		/**
		 * Get an input item by its order in the buffer.
		 *
		 * @param traverser
		 * @param id
		 * @returns {*}
		 */
		function getInputItemById (traverser, id) {
			return traverser.inputBuffer[id];
		}

		/**
		 * Returns current size of the input buffer.
		 *
		 * @param traverser
		 * @returns {Number}
		 */
		function getInputBufferSize (traverser) {
			return traverser.inputBuffer.length;
		}

		/**
		 * Reset the input buffer.
		 *
		 * @param traverser
		 */
		function resetInputBuffer (traverser) {
			// Reset the input buffer
			traverser.inputBuffer = [];

			// Reset the input over indicator
			traverser.inputOver = false;
		}

		/**
		 * Reset the final records
		 *
		 * @param traverser
		 */
		function resetFinalRecords (traverser) {
			traverser.finalRecords = [];
		}

		/**
		 * Get the list of final records.
		 *
		 * @param traverser
		 */
		function getFinalRecords (traverser) {
			return traverser.finalRecords;
		}

		/**
		 * Reset tail records to re-run the traverser.
		 *
		 * @param traverser
		 */
		function resetTailRecords (traverser) {

			// Create an initial record
			var initialRecord = createInitialRecord(traverser);

			// Update tail records
			updateTailRecords(traverser, [initialRecord]);
		}

		/**
		 * Replace old tail records with new tail records.
		 *
		 * @param traverser
		 * @param newTailRecords
		 */
		function updateTailRecords (traverser, newTailRecords) {
			traverser.tailRecords = newTailRecords;
		}

		/**
		 * Reset the traverser.
		 *
		 * @param traverser
		 */
		function reset (traverser) {

			// Reset the input buffer
			resetInputBuffer(traverser);

			// Reset the final records
			resetFinalRecords(traverser);

			// Reset the tail records
			resetTailRecords(traverser);
		}

		/**
		 * Check whether a given state is final.
		 *
		 * @param traverser
		 * @param state
		 * @returns {boolean}
		 */
		function isStateFinal (traverser, state) {
			return (-1 < traverser.finalStates.indexOf(state));
		}

		/**
		 * Creates an initial record.
		 *
		 * @param traverser
		 * @returns {Record}
		 */
		function createInitialRecord (traverser) {

			/*
				Initial record has null as its previous state declaring the recording beginning.
				At the same time, its target state is set to initial state. That is what we draw in graphs as '-->(0)'.
				It has an empty transition on it, since it doesn't have to consume any input.
				Finally, its accepting flag is set to 'true'. It is an important factor, as initial record
				takes part in cutting off the useless extensions when those are discovered at the very beginning of the input.
			 */
			return new Record(
				null,
				traverser.initialState,
				[''],
				true
			);
		}

		/**
		 * Create and return a new accept record.
		 *
		 * @param previousRecord
		 * @param character
		 * @param targetState
		 * @returns {Record}
		 */
		function createAcceptRecord (previousRecord, character, targetState) {

			// Return the new accept record
			return new Record (
				previousRecord,
				targetState,
				[character],
				true
			);
		}

		/**
		 * Create and return a new accepted record.
		 *
		 * @param previousRecord
		 * @param characters
		 * @param targetState
		 * @returns {Record}
		 */
		function createMissingRecord (previousRecord, characters, targetState) {

			// Return the new accepted record
			return new Record (
				previousRecord,
				targetState,
				characters,
				false
			);

		}

		/**
		 * Create a partially accepted record.
		 *
		 * @param previousRecord
		 * @param characters
		 * @param excludedCharacter
		 * @param targetState
		 * @returns {Record}
		 */
		function createPartiallyMissingRecord (previousRecord, characters, excludedCharacter, targetState) {

			// Save the next state index in the current transported transition
			var excludedCharacterIndex = characters.indexOf(excludedCharacter);

			// Create array of accepted characters
			var partialCharacters = characters.slice();

			// Cut off the accepted character
			partialCharacters.splice(excludedCharacterIndex, 1);

			// Return the new accepted record
			return createMissingRecord(
				previousRecord,
				partialCharacters,
				targetState
			);

		}

		/**
		 * Get transported transitions for a state.
		 *
		 * @param traverser
		 * @param currentState
		 * @returns {*}
		 */
		function getStateTransportedTransitions (traverser, currentState) {
			return traverser.transportedTransitions[currentState];
		}

		/**
		 * Get the next state for a given state and input.
		 *
		 * @param traverser
		 * @param currentState
		 * @param inputItem
		 * @returns {*}
		 */
		function getNextState (traverser, currentState, inputItem) {

			return traverser.transitions[currentState][inputItem];
		}

		/**
		 * Save record as final.
		 *
		 * @param traverser
		 * @param record
		 */
		function saveFinalRecord (traverser, record) {
			traverser.finalRecords.push(record);
		}

		/**
		 * From original Whynot
		 * Find proper position to insert a new record
		 *
		 * @param currentTails
		 * @param missingCount
		 * @returns {number}
		 */
		function findInsertionIndex (currentTails, missingCount) {
			// Perform a binary search to find the index of the first thread with lower badness
			var low = 0,
				high = currentTails.length;

			while (low < high) {
				// Use zero-filling shift as integer division
				var mid = (low + high) >>> 1;
				// Compare to mid point, preferring right in case of equality
				if (missingCount < currentTails[mid].getMissingCount()) {
					// Thread goes in lower half
					high = mid;
				} else {
					// Thread goes in upper half
					low = mid + 1;
				}
			}

			return low;
		}

		/**
		 * Insert a newly generated tail record to the latest tail records level
		 *
		 * @param tailRecords
		 * @param newTailRecord
		 */
		function insertNewTailRecord (tailRecords, newTailRecord) {

			var insertionIndex = findInsertionIndex(tailRecords, newTailRecord.getMissingCount());

			tailRecords.splice(insertionIndex, 0, newTailRecord);
		}

		function insertNewRecord (records, recordsIndex, newRecord) {

			var isAlternative = true;

			var recordsIndexLine = recordsIndex[newRecord.getTargetState()];

			for (var currentRecordId = 0; currentRecordId < recordsIndexLine.length; ++ currentRecordId) {
				var currentRecord = recordsIndexLine[currentRecordId];

				if (newRecord.isExtensionOf(currentRecord)) {
					// Is an extension of an existing tail record
					isAlternative = false;

					break;
				}
			}

			// Check for loops
			// ALSO check for EXTENSIONS
			if ((isAlternative) && (!newRecord.hasLoops())) {
				records.push(newRecord);
				recordsIndexLine.push(newRecord);
			}
		}

		/**
		 * Executes all the current tails and returns the new tails.
		 *
		 * @param traverser
		 * @param inputItem
		 * @param tailRecords
		 * @returns {Array}
		 */
		function processTailRecords (traverser, inputItem, tailRecords) {

			// Create new tail records array<commit_msg>Get rid of extra methods and properties in Traverser<commit_after>/**
 * Created by danek_000 on 30.8.2015.
 *
 *   .::::::::..          ..::::::::.
 * 	:::::::::::::        :::::::::::::
 * :::::::::::' .\      /. `:::::::::::
 * `::::::::::_,__o    o__,_::::::::::'
 */
define(
	[
		'./Record',
		'./util/arrayUtils'
	],
	function(
		Record
	) {
		'use strict';

		/**
		 * Describes a traverser to execute a DFA suggesting input expansion if required.
		 *
		 * @param biverseDFA
		 * @constructor
		 */
		function Traverser (biverseDFA) {

			// Define an initial state
			this.initialState = biverseDFA.initialState;

			// Define the conventional transitions table
			this.transitions = biverseDFA.transitions;

			// Define the transported transitions table
			this.transportedTransitions = transportTransitions(biverseDFA.transitions);

			// Define the final states
			this.finalStates = biverseDFA.finalStates;

			// Define the final records
			this.finalRecords = [];
		}

		/**
		 * Create transported transition table out of a given transition table.
		 *
		 * @param transitions
		 * @returns {Array}
		 */
		function transportTransitions (transitions) {

			var transportedTransitions = [];

			var statesCount = transitions.length;

			for (var stateNumber = 0; stateNumber < statesCount; stateNumber ++) {
				transportedTransitions[stateNumber] = {};

				var stateTransitionKeys = Object.keys(transitions[stateNumber]);

				var stateTransitionKeysCount = stateTransitionKeys.length;

				for (var stateTransitionKeyId = 0; stateTransitionKeyId < stateTransitionKeysCount; stateTransitionKeyId ++) {
					var stateTransitionKey = stateTransitionKeys[stateTransitionKeyId];

					var transition = transitions[stateNumber][stateTransitionKey];

					var transitionString = transition + '';

					if (transportedTransitions[stateNumber][transitionString] === undefined) {
						transportedTransitions[stateNumber][transitionString] = [];
					}

					transportedTransitions[stateNumber][transitionString].push(stateTransitionKey);
				}
			}

			return transportedTransitions;
		}

		/**
		 * Reset the final records
		 *
		 * @param traverser
		 */
		function resetFinalRecords (traverser) {
			traverser.finalRecords = [];
		}

		/**
		 * Get the list of final records.
		 *
		 * @param traverser
		 */
		function getFinalRecords (traverser) {
			return traverser.finalRecords;
		}

		/**
		 * Reset tail records to re-run the traverser.
		 *
		 * @param traverser
		 */
		function resetTailRecords (traverser) {

			// Create an initial record
			var initialRecord = createInitialRecord(traverser);

			// Update tail records
			updateTailRecords(traverser, [initialRecord]);
		}

		/**
		 * Replace old tail records with new tail records.
		 *
		 * @param traverser
		 * @param newTailRecords
		 */
		function updateTailRecords (traverser, newTailRecords) {
			traverser.tailRecords = newTailRecords;
		}

		/**
		 * Reset the traverser.
		 *
		 * @param traverser
		 */
		function reset (traverser) {

			// Reset the final records
			resetFinalRecords(traverser);

			// Reset the tail records
			resetTailRecords(traverser);
		}

		/**
		 * Check whether a given state is final.
		 *
		 * @param traverser
		 * @param state
		 * @returns {boolean}
		 */
		function isStateFinal (traverser, state) {
			return (-1 < traverser.finalStates.indexOf(state));
		}

		/**
		 * Creates an initial record.
		 *
		 * @param traverser
		 * @returns {Record}
		 */
		function createInitialRecord (traverser) {

			/*
				Initial record has null as its previous state declaring the recording beginning.
				At the same time, its target state is set to initial state. That is what we draw in graphs as '-->(0)'.
				It has an empty transition on it, since it doesn't have to consume any input.
				Finally, its accepting flag is set to 'true'. It is an important factor, as initial record
				takes part in cutting off the useless extensions when those are discovered at the very beginning of the input.
			 */
			return new Record(
				null,
				traverser.initialState,
				[''],
				true
			);
		}

		/**
		 * Create and return a new accept record.
		 *
		 * @param previousRecord
		 * @param character
		 * @param targetState
		 * @returns {Record}
		 */
		function createAcceptRecord (previousRecord, character, targetState) {

			// Return the new accept record
			return new Record (
				previousRecord,
				targetState,
				[character],
				true
			);
		}

		/**
		 * Create and return a new accepted record.
		 *
		 * @param previousRecord
		 * @param characters
		 * @param targetState
		 * @returns {Record}
		 */
		function createMissingRecord (previousRecord, characters, targetState) {

			// Return the new accepted record
			return new Record (
				previousRecord,
				targetState,
				characters,
				false
			);

		}

		/**
		 * Create a partially accepted record.
		 *
		 * @param previousRecord
		 * @param characters
		 * @param excludedCharacter
		 * @param targetState
		 * @returns {Record}
		 */
		function createPartiallyMissingRecord (previousRecord, characters, excludedCharacter, targetState) {

			// Save the next state index in the current transported transition
			var excludedCharacterIndex = characters.indexOf(excludedCharacter);

			// Create array of accepted characters
			var partialCharacters = characters.slice();

			// Cut off the accepted character
			partialCharacters.splice(excludedCharacterIndex, 1);

			// Return the new accepted record
			return createMissingRecord(
				previousRecord,
				partialCharacters,
				targetState
			);

		}

		/**
		 * Get transported transitions for a state.
		 *
		 * @param traverser
		 * @param currentState
		 * @returns {*}
		 */
		function getStateTransportedTransitions (traverser, currentState) {
			return traverser.transportedTransitions[currentState];
		}

		/**
		 * Get the next state for a given state and input.
		 *
		 * @param traverser
		 * @param currentState
		 * @param inputItem
		 * @returns {*}
		 */
		function getNextState (traverser, currentState, inputItem) {

			return traverser.transitions[currentState][inputItem];
		}

		/**
		 * Save record as final.
		 *
		 * @param traverser
		 * @param record
		 */
		function saveFinalRecord (traverser, record) {
			traverser.finalRecords.push(record);
		}

		/**
		 * From original Whynot
		 * Find proper position to insert a new record
		 *
		 * @param currentTails
		 * @param missingCount
		 * @returns {number}
		 */
		function findInsertionIndex (currentTails, missingCount) {
			// Perform a binary search to find the index of the first thread with lower badness
			var low = 0,
				high = currentTails.length;

			while (low < high) {
				// Use zero-filling shift as integer division
				var mid = (low + high) >>> 1;
				// Compare to mid point, preferring right in case of equality
				if (missingCount < currentTails[mid].getMissingCount()) {
					// Thread goes in lower half
					high = mid;
				} else {
					// Thread goes in upper half
					low = mid + 1;
				}
			}

			return low;
		}

		/**
		 * Insert a newly generated tail record to the latest tail records level
		 *
		 * @param tailRecords
		 * @param newTailRecord
		 */
		function insertNewTailRecord (tailRecords, newTailRecord) {

			var insertionIndex = findInsertionIndex(tailRecords, newTailRecord.getMissingCount());

			tailRecords.splice(insertionIndex, 0, newTailRecord);
		}

		function insertNewRecord (records, recordsIndex, newRecord) {

			var isAlternative = true;

			var recordsIndexLine = recordsIndex[newRecord.getTargetState()];

			for (var currentRecordId = 0; currentRecordId < recordsIndexLine.length; ++ currentRecordId) {
				var currentRecord = recordsIndexLine[currentRecordId];

				if (newRecord.isExtensionOf(currentRecord)) {
					// Is an extension of an existing tail record
					isAlternative = false;

					break;
				}
			}

			if ((isAlternative) && (!newRecord.hasLoops())) {
				records.push(newRecord);
				recordsIndexLine.push(newRecord);
			}
		}

		/**
		 * Executes all the current tails and returns the new tails.
		 *
		 * @param traverser
		 * @param inputItem
		 * @param tailRecords
		 * @returns {Array}
		 */
		function processTailRecords (traverser, inputItem, tailRecords) {

			// Create new tail records array
{'fraction_non_alphanumeric': 0.10484048404840485, 'fraction_numerical': 0.0022002200220022, 'mean_word_length': 2.9506736201651456, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4976227', 'n_tokens_mistral': 6435, 'n_tokens_neox': 4931, 'n_words': 1526}