<commit_before>package com.algorithms.huffman;

import java.util.*;
import java.io.*;

public class Huffman {

	public static void main (String args[]) {

		String folder1 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\speechdata";
		String folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent1";
		String outFile = "recent1.txt";
		processFiles(folder1, folder2, outFile);
		
		/*folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent2";
		outFile = "recent2.txt";
		processFiles(folder1, folder2, outFile);
		
		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent10";
		outFile = "recent10.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent100";
		outFile = "recent100.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent300";
		outFile = "recent300.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old2";
		outFile = "old2.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old10";
		outFile = "old10.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old100";
		outFile = "old100.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old300";
		outFile = "old300.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\speechdata";
		outFile = "all.out";
		processFiles(folder1, folder2, outFile);*/

	}


	/*
	 * folderName1 : folder containing all the speeches
	 * folderName2 : folder containing subset of speeches used to build the Huffman Code
	 */
	public static void processFiles (String folderName1, String folderName2, String outFileName)      {
		Scanner stdin = null;
		// map to hold words and frequencies of current set of speeches
		BSTDictionary<KeyWord> dictionary = new BSTDictionary<KeyWord>();
		// huffman tree made based on given subset of speeches
		HuffmanTree<String> tree;
		// map to hold the number of bits that represent each word in huffman
		HashMap<String, Integer> bitsPerWord = new HashMap<String, Integer>();
		// folder containing all speeches to be compressed
		File folder1 = new File(folderName1);
		File[] allSpeeches = folder1.listFiles();

		// process all files in speech directory
		for (File inFile : allSpeeches) {
			// check if file exits and can be read
			if (!inFile.exists() || !inFile.canRead()) {
				System.out.println("Improper file: " + inFile.getName());
				System.exit(-1);
			}//end if
			try {
				stdin = new Scanner(inFile);
				// add all words from all speeches with a frequency of 1
				smooth(dictionary, stdin);
			} catch (FileNotFoundException ex) {
				System.out.println("Unable to find file: " + inFile.getName());
				System.exit(-1);
			}//end try-catch
		}//end for

		double huffmanComp = 0;
		double blockComp = 0;
		int numWords = 0;
		// build huffman code using subset of files
		File folder2 = new File(folderName2);
		File[] listOfFiles = folder2.listFiles();
		// process subset of files to make huffman code
		for (File inFile : listOfFiles) {
			System.out.println(inFile.getName());
			if (!inFile.exists() || !inFile.canRead()) {
				System.out.println("Improper file: " + inFile.getName());
				System.exit(-1);
			}//end if
			try {
				stdin = new Scanner(inFile);
				parseFile(dictionary, stdin);

			} catch (FileNotFoundException ex) {
				System.out.println("Unable to find file: " + inFile.getName());
				System.exit(-1);
			}// end try-catch
		} //end-for
		
		tree = makeHuffman(dictionary);
		bitsPerWord = findBits(tree);
			
		for (File speech : allSpeeches) {
			// calculate compression ratio for all speeches
			try {
				stdin = new Scanner(speech);
				numWords = countWords(stdin);
				stdin = new Scanner(speech);
			} catch(FileNotFoundException e) {
				System.out.println("Unable to find fine: " + speech.getName());
			}
			huffmanComp = calcHuffmanCompression(stdin, bitsPerWord);
			blockComp = calcBlockCompression(dictionary, numWords);
			double ratio = huffmanComp/blockComp;
			try {
				File outFile = new File(outFileName);
				PrintWriter out = new PrintWriter(outFile);
				out.append("\n" + ratio);
			} catch (IOException e) {
				System.out.println("Could not write to file : " + speech.getName());
			} 
		} //end-for
	}


	/*
	 * Creates a mapping of words to the number of bits needed to encode them
	 */
	public static HashMap<String, Integer> findBits(HuffmanTree<String> tree) {
		HashMap<String, Integer> map = new HashMap<String, Integer>();
		Queue<HuffmanNode<String>> queue = new LinkedList<HuffmanNode<String>>();
		queue.add(tree.getRoot());
		tree.getRoot().setVisited(true);
		while (!queue.isEmpty()) {
			HuffmanNode<String> node = queue.remove();
			map.put(node.getData(), node.getHeight());
			if (node.leftChild != null && !node.leftChild.visited) {
				node.leftChild.setVisited(true);
				queue.add(node.leftChild);
			}
			if (node.rightChild != null && !node.rightChild.visited) {
				node.rightChild.setVisited(true);
				queue.add(node.rightChild);
			}
		}
		return map;
	}
<commit_msg>still working on outputting to file, not working<commit_after>package com.algorithms.huffman;

import java.util.*;
import java.io.*;

public class Huffman {

	public static void main (String args[]) {

		String folder1 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\speechdata";
		String folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent1";
		File outFile = new File("recent1.txt");
		processFiles(folder1, folder2, outFile);
		
		/*folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent2";
		outFile = "recent2.txt";
		processFiles(folder1, folder2, outFile);
		
		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent10";
		outFile = "recent10.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent100";
		outFile = "recent100.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\recent300";
		outFile = "recent300.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old2";
		outFile = "old2.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old10";
		outFile = "old10.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old100";
		outFile = "old100.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\old300";
		outFile = "old300.out";
		processFiles(folder1, folder2, outFile);

		folder2 = "\\Users\\Kristin\\Documents\\GitHub\\CS577-Huffman\\Huffman\\speechdata";
		outFile = "all.out";
		processFiles(folder1, folder2, outFile);*/

	}


	/*
	 * folderName1 : folder containing all the speeches
	 * folderName2 : folder containing subset of speeches used to build the Huffman Code
	 */
	public static void processFiles (String folderName1, String folderName2, File outFileName)      {
		Scanner stdin = null;
		// map to hold words and frequencies of current set of speeches
		BSTDictionary<KeyWord> dictionary = new BSTDictionary<KeyWord>();
		// huffman tree made based on given subset of speeches
		HuffmanTree<String> tree;
		// map to hold the number of bits that represent each word in huffman
		HashMap<String, Integer> bitsPerWord = new HashMap<String, Integer>();
		// folder containing all speeches to be compressed
		File folder1 = new File(folderName1);
		File[] allSpeeches = folder1.listFiles();

		// process all files in speech directory
		for (File inFile : allSpeeches) {
			// check if file exits and can be read
			if (!inFile.exists() || !inFile.canRead()) {
				System.out.println("Improper file: " + inFile.getName());
				System.exit(-1);
			}//end if
			try {
				stdin = new Scanner(inFile);
				// add all words from all speeches with a frequency of 1
				smooth(dictionary, stdin);
			} catch (FileNotFoundException ex) {
				System.out.println("Unable to find file: " + inFile.getName());
				System.exit(-1);
			}//end try-catch
		}//end for

		double huffmanComp = 0;
		double blockComp = 0;
		int numWords = 0;
		// build huffman code using subset of files
		File folder2 = new File(folderName2);
		File[] listOfFiles = folder2.listFiles();
		// process subset of files to make huffman code
		for (File inFile : listOfFiles) {
			System.out.println(inFile.getName());
			if (!inFile.exists() || !inFile.canRead()) {
				System.out.println("Improper file: " + inFile.getName());
				System.exit(-1);
			}//end if
			try {
				stdin = new Scanner(inFile);
				parseFile(dictionary, stdin);

			} catch (FileNotFoundException ex) {
				System.out.println("Unable to find file: " + inFile.getName());
				System.exit(-1);
			}// end try-catch
		} //end-for
		
		tree = makeHuffman(dictionary);
		bitsPerWord = findBits(tree);

		try {
			PrintWriter out = new PrintWriter(outFileName);
			for (File speech : allSpeeches) {
				// calculate compression ratio for all speeches
				try {
					stdin = new Scanner(speech);
					numWords = countWords(stdin);
					stdin = new Scanner(speech);
				} catch(FileNotFoundException e) {
					System.out.println("Unable to find fine: " + speech.getName());
				}
				huffmanComp = calcHuffmanCompression(stdin, bitsPerWord);
				blockComp = calcBlockCompression(dictionary, numWords);
				double ratio = huffmanComp/blockComp;
				out.append("\n" + ratio);

			} //end-for
		} catch (IOException e) {
			System.out.println("Could not write to file");
		} 
	}


	/*
	 * Creates a mapping of words to the number of bits needed to encode them
	 */
	public static HashMap<String, Integer> findBits(HuffmanTree<String> tree) {
		HashMap<String, Integer> map = new HashMap<String, Integer>();
		Queue<HuffmanNode<String>> queue = new LinkedList<HuffmanNode<String>>();
		queue.add(tree.getRoot());
		tree.getRoot().setVisited(true);
		while (!queue.isEmpty()) {
			HuffmanNode<String> node = queue.remove();
			map.put(node.getData(), node.getHeight());
			if (node.leftChild != null && !node.leftChild.visited) {
				node.leftChild.setVisited(true);
				queue.add(node.leftChild);
			}
			if (node.rightChild != null && !node.rightChild.visited) {
				node.rightChild.setVisited(true);
				queue.add(node.rightChild);
			}
		}
		return map;
	}

{'fraction_non_alphanumeric': 0.14439042289698192, 'fraction_numerical': 0.022016328777176405, 'mean_word_length': 3.5787484250314994, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5892381', 'n_tokens_mistral': 4226, 'n_tokens_neox': 3516, 'n_words': 829}