<commit_before>//to access filesystem
const fs = require('fs');
//to format filepath
const path = require('path');

//load private Utils
var commonUtil = require("../common/commonUtil");
var commonSearchUtil = require("../common/commonSearchUtil");
var searchDriveUtil = require("../complementPathUtils/searchDriveUtil");
var complementPathUtil = require("../complementPathUtils/complementPathUtil");

//to manage progress
var progressManager = require("./searchProgressManager");
//to manage sorting
var searchSortManager = require("./searchSortManager");


// param : "a bc  "
// return: [1,4,5]
function checkPositionOfSpaces(targetString){
	var positionOfSpaces = [];
	for(var index = 0 , len = targetString.length ; index < len; index++){
		if((targetString[index] == ' ') || (targetString[index] == '　' )){
			positionOfSpaces.push(index);
		}
	}
	return positionOfSpaces;
}

//inner function for recursive search
// param : [], "A A A"
// return:
// process: [] <- ["A A A","AA A","A AA","AAA"]
function innerRecursiveListUp(list, string){
	if(list.indexOf(string) < 0){
		list.push(string);
	}
	var positionOfSpaces = checkPositionOfSpaces(string);
	for(var index = 0, len = positionOfSpaces.length; index < len; index++){
		var stringRemovedOneSpace = commonUtil.removeCharacterWithPosition(string, positionOfSpaces[index]);
		innerRecursiveListUp(list, stringRemovedOneSpace);
	}
}

// param : "A A A"
// return: ["A A A","AA A","A AA","AAA"]
function listupSearchCandidates(string){
	var list = [];
	innerRecursiveListUp(list, string);
	return list;
}

//inner function
// param : "A A A\B BB\C  C"
// return: [["A A A","AA A","A AA","AAA"],
//          ["B BB","BBB"],
//          ["C  C","C C","CC"]]
function listupAllLayerSearchCandidates(path){
	var listAllLayer = [];
	var separatedPath = commonSearchUtil.separatePath(path, true);
	for(var index = 0, len = separatedPath.length; index < len; index++){
		listAllLayer.push(listupSearchCandidates(separatedPath[index]));
	}
	return listAllLayer;
}

//add to res
function addOpenCommand(targetPath, state, res){
	//add to res.
	//Check state of formatted path (File or Folder or not).
	//and set Description Message according to the state.
	var descriptionMessage = "";

	var availableFullPath = targetPath;
	//TODO: calc distance
	var distance = "X";

	var addToResFlag = false;
	var innerIcon = "";
	var innerGroup = "";
	var innerRedirect = "";
	switch(state){
		case -1://invalid
		case 0://invalid
			//descriptionMessage = "Not File/Folder. Cannot open."
			break;
		case "file"://file
			//extract file name
			//(todo) extract BBB from "C:\AAA\BBB\" <- when unnecessary "\" exists.
			var filename = availableFullPath.slice().split(path.sep).pop();
			descriptionMessage = "Open this File : \"" + filename +
				"\" ( Distance = " + distance + " )";
			innerIcon = "#fa fa-file-o";
			addToResFlag = true;
			innerGroup = "Available Pathes : File";
			innerRedirect = commonUtil.commandHeader + " " + availableFullPath;
			break;
		case "folder"://folder
			//extract folder name
			//(todo) extract BBB from "C:\AAA\BBB\" <- when unnecessary "\" exists.
			var foldername = availableFullPath.slice().split(path.sep).pop();
			descriptionMessage = "Open this Folder : \"" + foldername +
				"\" ( Distance = " + distance + " )";
			innerIcon = "#fa fa-folder-open-o";
			addToResFlag = true;
			innerGroup = "Available Pathes : Folder";
			innerRedirect = commonUtil.commandHeader + " " + availableFullPath + "\\";
			break;
		case "drive"://file server
			//extract folder name
			var foldername = availableFullPath.slice().split(path.sep).pop();
			descriptionMessage = "Open this Drive : \"" + foldername +
				"\" ( Distance = " + distance + " )";
			//innerIcon = "#fa fa-folder-open-o";
			innerIcon = "#fa fa-server";
			addToResFlag = true;
			innerGroup = "Available Pathes : Drive";
			innerRedirect = commonUtil.commandHeader + " " + availableFullPath + "\\";
			break;
		default:
			break;
	}
	//add to res.
	if(addToResFlag == true){
		res.add(
			{
				id: availableFullPath,
				payload: 'open',
				title: availableFullPath,
				icon: innerIcon,
				desc: descriptionMessage,
				redirect: innerRedirect,
				group: innerGroup
			}
		);
	}
}

//inner function for recursive search
//add to res in this function
//TODO: calc difference
function innerRecursiveSearch(path, searchedPathes, currentPath, listRemainingLayer, state, res){
	//main process
	var len = listRemainingLayer.length;
	if(len > 0){
		var list = listRemainingLayer[0];
		let copy = listRemainingLayer.slice();
		copy.shift();
		for(var index = 0, len2 = list.length; index < len2; index++){
			let target = "";
			if(currentPath != ""){
				target = target + currentPath + "\\";
			}
			target = target + list[index];
			fs.stat(target, function(err, stats){
				if(err){
					//console.log("err");
					progressManager.addProgressByRemainingList(copy);
					//execute complement of path
				}else{
					if(stats.isFile() == true){
						innerRecursiveSearch(path, searchedPathes, target ,copy, "file", res);
					}else if(stats.isDirectory() == true){
						//TODO: identify folder or fileserver or drive
						innerRecursiveSearch(path, searchedPathes, target ,copy, "folder", res);
					}
				}
				//check progress
				checkProgress(path, res);
			});
		}
	}else{// len == 0
		if(searchedPathes.indexOf(currentPath) < 0){
			searchedPathes.push(currentPath);
			//callback(currentPath, res);
			//addOpenCommand(currentPath, res);
			//add progress
			progressManager.addProgressByNum(1);
			progressManager.addFoundPathNum();
			//add found path
			searchSortManager.add(currentPath, state, 0);<commit_msg>remove unnecessary comment out<commit_after>//to access filesystem
const fs = require('fs');
//to format filepath
const path = require('path');

//load private Utils
var commonUtil = require("../common/commonUtil");
var commonSearchUtil = require("../common/commonSearchUtil");
var searchDriveUtil = require("../complementPathUtils/searchDriveUtil");
var complementPathUtil = require("../complementPathUtils/complementPathUtil");

//to manage progress
var progressManager = require("./searchProgressManager");
//to manage sorting
var searchSortManager = require("./searchSortManager");


// param : "a bc  "
// return: [1,4,5]
function checkPositionOfSpaces(targetString){
	var positionOfSpaces = [];
	for(var index = 0 , len = targetString.length ; index < len; index++){
		if((targetString[index] == ' ') || (targetString[index] == '　' )){
			positionOfSpaces.push(index);
		}
	}
	return positionOfSpaces;
}

//inner function for recursive search
// param : [], "A A A"
// return:
// process: [] <- ["A A A","AA A","A AA","AAA"]
function innerRecursiveListUp(list, string){
	if(list.indexOf(string) < 0){
		list.push(string);
	}
	var positionOfSpaces = checkPositionOfSpaces(string);
	for(var index = 0, len = positionOfSpaces.length; index < len; index++){
		var stringRemovedOneSpace = commonUtil.removeCharacterWithPosition(string, positionOfSpaces[index]);
		innerRecursiveListUp(list, stringRemovedOneSpace);
	}
}

// param : "A A A"
// return: ["A A A","AA A","A AA","AAA"]
function listupSearchCandidates(string){
	var list = [];
	innerRecursiveListUp(list, string);
	return list;
}

//inner function
// param : "A A A\B BB\C  C"
// return: [["A A A","AA A","A AA","AAA"],
//          ["B BB","BBB"],
//          ["C  C","C C","CC"]]
function listupAllLayerSearchCandidates(path){
	var listAllLayer = [];
	var separatedPath = commonSearchUtil.separatePath(path, true);
	for(var index = 0, len = separatedPath.length; index < len; index++){
		listAllLayer.push(listupSearchCandidates(separatedPath[index]));
	}
	return listAllLayer;
}

//add to res
function addOpenCommand(targetPath, state, res){
	//add to res.
	//Check state of formatted path (File or Folder or not).
	//and set Description Message according to the state.
	var descriptionMessage = "";

	var availableFullPath = targetPath;
	//TODO: calc distance
	var distance = "X";

	var addToResFlag = false;
	var innerIcon = "";
	var innerGroup = "";
	var innerRedirect = "";
	switch(state){
		case -1://invalid
		case 0://invalid
			//descriptionMessage = "Not File/Folder. Cannot open."
			break;
		case "file"://file
			//extract file name
			//(todo) extract BBB from "C:\AAA\BBB\" <- when unnecessary "\" exists.
			var filename = availableFullPath.slice().split(path.sep).pop();
			descriptionMessage = "Open this File : \"" + filename +
				"\" ( Distance = " + distance + " )";
			innerIcon = "#fa fa-file-o";
			addToResFlag = true;
			innerGroup = "Available Pathes : File";
			innerRedirect = commonUtil.commandHeader + " " + availableFullPath;
			break;
		case "folder"://folder
			//extract folder name
			//(todo) extract BBB from "C:\AAA\BBB\" <- when unnecessary "\" exists.
			var foldername = availableFullPath.slice().split(path.sep).pop();
			descriptionMessage = "Open this Folder : \"" + foldername +
				"\" ( Distance = " + distance + " )";
			innerIcon = "#fa fa-folder-open-o";
			addToResFlag = true;
			innerGroup = "Available Pathes : Folder";
			innerRedirect = commonUtil.commandHeader + " " + availableFullPath + "\\";
			break;
		case "drive"://file server
			//extract folder name
			var foldername = availableFullPath.slice().split(path.sep).pop();
			descriptionMessage = "Open this Drive : \"" + foldername +
				"\" ( Distance = " + distance + " )";
			//innerIcon = "#fa fa-folder-open-o";
			innerIcon = "#fa fa-server";
			addToResFlag = true;
			innerGroup = "Available Pathes : Drive";
			innerRedirect = commonUtil.commandHeader + " " + availableFullPath + "\\";
			break;
		default:
			break;
	}
	//add to res.
	if(addToResFlag == true){
		res.add(
			{
				id: availableFullPath,
				payload: 'open',
				title: availableFullPath,
				icon: innerIcon,
				desc: descriptionMessage,
				redirect: innerRedirect,
				group: innerGroup
			}
		);
	}
}

//inner function for recursive search
//add to res in this function
//TODO: calc difference
function innerRecursiveSearch(path, searchedPathes, currentPath, listRemainingLayer, state, res){
	//main process
	var len = listRemainingLayer.length;
	if(len > 0){
		var list = listRemainingLayer[0];
		let copy = listRemainingLayer.slice();
		copy.shift();
		for(var index = 0, len2 = list.length; index < len2; index++){
			let target = "";
			if(currentPath != ""){
				target = target + currentPath + "\\";
			}
			target = target + list[index];
			fs.stat(target, function(err, stats){
				if(err){
					//console.log("err");
					progressManager.addProgressByRemainingList(copy);
					//execute complement of path
				}else{
					if(stats.isFile() == true){
						innerRecursiveSearch(path, searchedPathes, target ,copy, "file", res);
					}else if(stats.isDirectory() == true){
						//TODO: identify folder or fileserver or drive
						innerRecursiveSearch(path, searchedPathes, target ,copy, "folder", res);
					}
				}
				//check progress
				checkProgress(path, res);
			});
		}
	}else{// len == 0
		if(searchedPathes.indexOf(currentPath) < 0){
			searchedPathes.push(currentPath);
			//add progress
			progressManager.addProgressByNum(1);
			progressManager.addFoundPathNum();
			//add found path
			searchSortManager.add(currentPath, state, 0);
{'fraction_non_alphanumeric': 0.15353248825458735, 'fraction_numerical': 0.003191206453328606, 'mean_word_length': 3.7126148705096074, 'pattern_counts': {'":': 6, '<': 21, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '657819', 'n_tokens_mistral': 4000, 'n_tokens_neox': 3517, 'n_words': 904}