<commit_before>package cleaner;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;

import main.Updater;

public class Lists
{
	public static String BLACKLIST = "Blacklist";
	public static String WHITELIST = "Whitelist";
	public static String EXTENSION = "Extension";
	public static String BLACKLISTFOLDER = "BFolder";
	public static String WHITELISTFOLDER = "WFolder";
	public static String HISTORY = "History";
	public static String CACHE = "Cache";
	public static String LOG = "Log";
	public static String UNKNOWN = "Uknown";
	
	public static File listsDir = new File("lists/");
	public static File logPath = new File("lists/"+LOG+".txt");
	public static MatchList extensions = new MatchList();	//these file extensions will be blacklisted
	public static MatchList blacklist = new MatchList();	//these files will be blacklisted
	public static MatchList whitelist = new MatchList();	//these files will be whitelisted
	public static MatchList blacklistFolders = new MatchList();	//files in these folders will be blacklisted
	public static MatchList whitelistFolders = new MatchList();	//files in these folders will be whitelisted
	public static MatchList cacheFolders = new MatchList();		//contains copies of recent files (files here will be checked)
	public static MatchList historyFolders = new MatchList();	//contains links to recent files (files here will be checked)
	
	public static ArrayList<ListFile> allLists = new ArrayList<ListFile>();
	
	public static boolean saveToLog = true;
	public static boolean deleteMatchedFiles = false;
	
	public static void main(String[] args)
	{
		loadLists();
		
//		for(String str : blacklist.getContents())
//			System.out.println(str);
		
//		ArrayList<Match> cacheMatches = getMatches(cacheFolders, true);
//		for(Match match : cacheMatches)
//			if(match.isBlacklisted)
//				System.out.println(match);
		
		ArrayList<Match> historyMatches = getMatches(historyFolders, true);
		for(Match match : historyMatches)
			if(match.isBlacklisted)
				System.out.println(match.toStringmaxChars(StringParser.getMaxChars(historyMatches)));
	}
	
	public static ArrayList<Match> getMatches(MatchList folders, boolean checkSubFolders)
	{
		long startTime = System.currentTimeMillis();
		
		ArrayList<Match> matches = new ArrayList<Match>();
		for(String str : folders.getUnrestrictedContents())		//for each folder path to check
		{
			ArrayList<File> files = new ArrayList<File>();
			if(checkSubFolders)
				files = SaveNLoad.getFilesRecur(new File(StringParser.addSpecialFolders(str)));	//check subfolders
			else
				files = SaveNLoad.getFiles(new File(StringParser.addSpecialFolders(str)));		//check outermost files
			
			Updater.setTotalFiles(files.size());
			
			for(int i = 0; i < files.size(); i++)	//for all the files in that folder
			{
				files.set(i, new File(StringParser.parseShortcut(files.get(i).getAbsolutePath())));
				
				Match match = getMatch(files.get(i));
				if(!match.isNull())			//if the file was matched
					matches.add(match);		//add it to the output
				
				Updater.incrementScannedFiles();
			}
		}
		
		//save to the log file
		if(saveToLog)
		{
			ArrayList<String> logStr = new ArrayList<String>();
			
			if(matches.size() > 0)	//if there were matches
			{
				//add the date, time and duration to the top
				String date = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(Calendar.getInstance().getTime());
				String duration = ((System.currentTimeMillis()-startTime))+" ms";
				logStr.add(date+" "+duration);
				
				//add a row of '~'
				String line = "";
				for(int i = 0; i < date.length()+duration.length()+1; i++)
					line += '~';
				logStr.add(line);
				
				//add the list of relevant matches
				int maxChars = StringParser.getMaxChars(matches);
				for(int i = 0; i < matches.size(); i++)	//for all the matches
				{
					Match match = matches.get(i);
					if(match.isBlacklisted)				//if the match was interesting
						logStr.add(match.toStringmaxChars(maxChars));	//add it to the list
				}
				
				logStr.add("");

				
				SaveNLoad.addArrayListToFile(logStr, logPath.getAbsolutePath());
			}
		}
		
		return matches;
	}
	
	//check all the files in the lists directory, and add them to their corresponding MatchLists
	public static void loadLists()
	{
		ArrayList<File> files = SaveNLoad.getFilesRecur(listsDir);
		for(File file : files)<commit_msg>Doesn't break if there is no lists folder<commit_after>package cleaner;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;

import main.Updater;

public class Lists
{
	public static String BLACKLIST = "Blacklist";
	public static String WHITELIST = "Whitelist";
	public static String EXTENSION = "Extension";
	public static String BLACKLISTFOLDER = "BFolder";
	public static String WHITELISTFOLDER = "WFolder";
	public static String HISTORY = "History";
	public static String CACHE = "Cache";
	public static String LOG = "Log";
	public static String UNKNOWN = "Uknown";
	
	public static File listsDir = new File("lists/");
	public static File logPath = new File("lists/"+LOG+".txt");
	public static MatchList extensions = new MatchList();	//these file extensions will be blacklisted
	public static MatchList blacklist = new MatchList();	//these files will be blacklisted
	public static MatchList whitelist = new MatchList();	//these files will be whitelisted
	public static MatchList blacklistFolders = new MatchList();	//files in these folders will be blacklisted
	public static MatchList whitelistFolders = new MatchList();	//files in these folders will be whitelisted
	public static MatchList cacheFolders = new MatchList();		//contains copies of recent files (files here will be checked)
	public static MatchList historyFolders = new MatchList();	//contains links to recent files (files here will be checked)
	
	public static ArrayList<ListFile> allLists = new ArrayList<ListFile>();
	
	public static boolean saveToLog = true;
	public static boolean deleteMatchedFiles = false;
	
	public static void main(String[] args)
	{
		loadLists();
		
//		for(String str : blacklist.getContents())
//			System.out.println(str);
		
//		ArrayList<Match> cacheMatches = getMatches(cacheFolders, true);
//		for(Match match : cacheMatches)
//			if(match.isBlacklisted)
//				System.out.println(match);
		
		ArrayList<Match> historyMatches = getMatches(historyFolders, true);
		for(Match match : historyMatches)
			if(match.isBlacklisted)
				System.out.println(match.toStringmaxChars(StringParser.getMaxChars(historyMatches)));
	}
	
	public static ArrayList<Match> getMatches(MatchList folders, boolean checkSubFolders)
	{
		long startTime = System.currentTimeMillis();
		
		ArrayList<Match> matches = new ArrayList<Match>();
		for(String str : folders.getUnrestrictedContents())		//for each folder path to check
		{
			ArrayList<File> files = new ArrayList<File>();
			if(checkSubFolders)
				files = SaveNLoad.getFilesRecur(new File(StringParser.addSpecialFolders(str)));	//check subfolders
			else
				files = SaveNLoad.getFiles(new File(StringParser.addSpecialFolders(str)));		//check outermost files
			
			Updater.setTotalFiles(files.size());
			
			for(int i = 0; i < files.size(); i++)	//for all the files in that folder
			{
				files.set(i, new File(StringParser.parseShortcut(files.get(i).getAbsolutePath())));
				
				Match match = getMatch(files.get(i));
				if(!match.isNull())			//if the file was matched
					matches.add(match);		//add it to the output
				
				Updater.incrementScannedFiles();
			}
		}
		
		//save to the log file
		if(saveToLog)
		{
			ArrayList<String> logStr = new ArrayList<String>();
			
			if(matches.size() > 0)	//if there were matches
			{
				//add the date, time and duration to the top
				String date = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(Calendar.getInstance().getTime());
				String duration = ((System.currentTimeMillis()-startTime))+" ms";
				logStr.add(date+" "+duration);
				
				//add a row of '~'
				String line = "";
				for(int i = 0; i < date.length()+duration.length()+1; i++)
					line += '~';
				logStr.add(line);
				
				//add the list of relevant matches
				int maxChars = StringParser.getMaxChars(matches);
				for(int i = 0; i < matches.size(); i++)	//for all the matches
				{
					Match match = matches.get(i);
					if(match.isBlacklisted)				//if the match was interesting
						logStr.add(match.toStringmaxChars(maxChars));	//add it to the list
				}
				
				logStr.add("");

				
				SaveNLoad.addArrayListToFile(logStr, logPath.getAbsolutePath());
			}
		}
		
		return matches;
	}
	
	//check all the files in the lists directory, and add them to their corresponding MatchLists
	public static void loadLists()
	{
		if(!listsDir.exists())
		{
			System.err.println("Lists folder not found!");
//			System.exit(0);
			return;
		}
		
		ArrayList<File> files = SaveNLoad.getFilesRecur(listsDir);
		for(File file : files)
{'fraction_non_alphanumeric': 0.11577068927057774, 'fraction_numerical': 0.001226857015391479, 'mean_word_length': 3.9188151398793196, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '872093', 'n_tokens_mistral': 3137, 'n_tokens_neox': 2684, 'n_words': 803}