<commit_before>#include <iostream>
#include <fstream>
#include <string>
#include <queue>
#include <vector>
#include <bitset>

using namespace std;

struct huffNode
{
	int frequency;
	char character;
	huffNode* left;
	huffNode* right;
};

struct node_comparison
{
	bool operator () (const huffNode* A, const huffNode* B) const
	{
		return A->frequency > B->frequency;
	}
};

void generateEncodings(huffNode *root, vector<bool> & encoding, vector<vector<bool>> & encodingTable)
{
	if (root != NULL) {
		if (root->character != -1)
		{
			encodingTable[root->character] = encoding;
		}
		vector<bool> leftEncoding = encoding;
		leftEncoding.push_back(false);
		vector<bool> rightEncoding = encoding;
		rightEncoding.push_back(true);

		generateEncodings(root->left,leftEncoding,encodingTable);
		generateEncodings(root->right,rightEncoding,encodingTable);
	}
}

void generateInitialPQueue(int frequencyList[256], priority_queue<huffNode*, vector<huffNode*>, node_comparison> &nodeHeap)
{
	for (int i = 0; i < 256; i++)
	{
		if (frequencyList[i] != 0)
		{
			huffNode* tempNode;
			tempNode = new huffNode;
			tempNode->frequency=frequencyList[i];
			tempNode->character = (char)i;
			tempNode->left = NULL;
			tempNode->right = NULL;
			nodeHeap.push(tempNode);
		}
	}
}

void printPQueue(priority_queue<huffNode*, vector<huffNode*>, node_comparison> &nodeHeap)
{
	if (nodeHeap.empty())
	{
		cout << "There's nothing in the heap!" << endl;
	}
	while (!nodeHeap.empty())
	{
		huffNode* tempNode;
		tempNode = nodeHeap.top();
		cout << tempNode->character << endl;
		nodeHeap.pop();
	}
}

void initializeFrequencyList(int frequencyList[256])
{
	for (int i = 0; i < 256; i++)
		frequencyList[i] = 0;
}

void generateFrequencyList(ifstream& fin, int frequencyList[256])
{
	string line = "";
	while (!fin.eof())
	{
		getline(fin,line);
		for (int i = 0; i < line.length(); i++)
		{
			frequencyList[line[i]] += 1;
		}
		frequencyList[10] += 1;
		line = "";
	}
}

void printFrequencyList(int frequencyList[256])
{
	for (int i = 0; i < 256; i++)
	{
		if (frequencyList[i] != 0)
		{
			cout << "Character " << (char)i << ": " << frequencyList[i] << endl;
		}
	}
}

void getEncoding(vector<bool> bitString, string &encoding)
{
	for (int i = 0; i < bitString.size(); i++)
	{
		encoding += bitString[i];
	}
}

void printEncodings(vector<vector<bool>> encodingTable)//vector<bool> encodingTable[256])
{
	string encoding = "";
	for (int i = 0; i < 256; i++)
	{
		if (!(encodingTable[i].empty()))
		{
			getEncoding(encodingTable[i], encoding);
			cout << "Character " << (char)i << " encoding: " << encoding << endl;
			encoding = "";
		}
	}
}

void generateHuffmanTree(priority_queue <huffNode*, vector<huffNode*>, node_comparison> &heap)
{
	if (heap.size() <= 1)
		return;
	else
	{
		huffNode *node1, *node2;
		node1 = new huffNode;
		node2 = new huffNode;

		node1 = heap.top();
		heap.pop();
		node2 = heap.top();
		heap.pop();
		int totalFreq = node1->frequency + node2->frequency;

		huffNode* newNode;
		newNode = new huffNode;
		newNode->frequency = totalFreq;
		newNode->character = (char)255;
		newNode->left = node1;
		newNode->right = node2;

		heap.push(newNode);
		totalFreq = 0;
		generateHuffmanTree(heap);
	}
}

void convertBuffer(vector<bool> & ogBuffer, bitset<8> &newBuffer)
{
	for (int i = 0; i < ogBuffer.size(); i++)
	{
		newBuffer[i] = ogBuffer[i];
	}
}

void checkBuffer(vector<bool> & bitBuffer, vector<bool> & encoding, ofstream &fout)
{
	while (encoding.size() != 0 )
	{
		if ((bitBuffer.size() + encoding.size()) >= 8) {
			int availableSpace = 8 - bitBuffer.size();
			for (int i = 0; i < availableSpace; i++)
			{
				bitBuffer.push_back(encoding[0]);
				encoding.erase(encoding.begin());
			}
			//convert vector<bool> to bitbuffer for output
			bitset<8> outputBuffer;
			convertBuffer(bitBuffer, outputBuffer);
			unsigned char n = outputBuffer.to_ulong();
			fout << n;
			bitBuffer.clear();
		}
		cout << "You're Stuck" << endl;
	}<commit_msg>Fixed bug when packing character encodings into bytes that was causing program to crash when encodings exceeded 8 bits.<commit_after>#include <iostream>
#include <fstream>
#include <string>
#include <queue>
#include <vector>
#include <bitset>

using namespace std;

struct huffNode
{
	int frequency;
	char character;
	huffNode* left;
	huffNode* right;
};

struct node_comparison
{
	bool operator () (const huffNode* A, const huffNode* B) const
	{
		return A->frequency > B->frequency;
	}
};

void generateEncodings(huffNode *root, vector<bool> & encoding, vector<vector<bool>> & encodingTable)
{
	if (root != NULL) {
		if (root->character != -1)
		{
			encodingTable[root->character] = encoding;
		}
		vector<bool> leftEncoding = encoding;
		leftEncoding.push_back(false);
		vector<bool> rightEncoding = encoding;
		rightEncoding.push_back(true);

		generateEncodings(root->left,leftEncoding,encodingTable);
		generateEncodings(root->right,rightEncoding,encodingTable);
	}
}

void generateInitialPQueue(int frequencyList[256], priority_queue<huffNode*, vector<huffNode*>, node_comparison> &nodeHeap)
{
	for (int i = 0; i < 256; i++)
	{
		if (frequencyList[i] != 0)
		{
			huffNode* tempNode;
			tempNode = new huffNode;
			tempNode->frequency=frequencyList[i];
			tempNode->character = (char)i;
			tempNode->left = NULL;
			tempNode->right = NULL;
			nodeHeap.push(tempNode);
		}
	}
}

void printPQueue(priority_queue<huffNode*, vector<huffNode*>, node_comparison> &nodeHeap)
{
	if (nodeHeap.empty())
	{
		cout << "There's nothing in the heap!" << endl;
	}
	while (!nodeHeap.empty())
	{
		huffNode* tempNode;
		tempNode = nodeHeap.top();
		cout << tempNode->character << endl;
		nodeHeap.pop();
	}
}

void initializeFrequencyList(int frequencyList[256])
{
	for (int i = 0; i < 256; i++)
		frequencyList[i] = 0;
}

void generateFrequencyList(ifstream& fin, int frequencyList[256])
{
	string line = "";
	while (!fin.eof())
	{
		getline(fin,line);
		for (int i = 0; i < line.length(); i++)
		{
			frequencyList[line[i]] += 1;
		}
		frequencyList[10] += 1;
		line = "";
	}
}

void printFrequencyList(int frequencyList[256])
{
	for (int i = 0; i < 256; i++)
	{
		if (frequencyList[i] != 0)
		{
			cout << "Character " << (char)i << ": " << frequencyList[i] << endl;
		}
	}
}

void getEncoding(vector<bool> bitString, string &encoding)
{
	for (int i = 0; i < bitString.size(); i++)
	{
		encoding += bitString[i];
	}
}

void printEncodings(vector<vector<bool>> encodingTable)//vector<bool> encodingTable[256])
{
	string encoding = "";
	for (int i = 0; i < 256; i++)
	{
		if (!(encodingTable[i].empty()))
		{
			getEncoding(encodingTable[i], encoding);
			cout << "Character " << (char)i << " encoding: " << encoding << endl;
			encoding = "";
		}
	}
}

void generateHuffmanTree(priority_queue <huffNode*, vector<huffNode*>, node_comparison> &heap)
{
	if (heap.size() <= 1)
		return;
	else
	{
		huffNode *node1, *node2;
		node1 = new huffNode;
		node2 = new huffNode;

		node1 = heap.top();
		heap.pop();
		node2 = heap.top();
		heap.pop();
		int totalFreq = node1->frequency + node2->frequency;

		huffNode* newNode;
		newNode = new huffNode;
		newNode->frequency = totalFreq;
		newNode->character = (char)255;
		newNode->left = node1;
		newNode->right = node2;

		heap.push(newNode);
		totalFreq = 0;
		generateHuffmanTree(heap);
	}
}

void convertBuffer(vector<bool> & ogBuffer, bitset<8> &newBuffer)
{
	for (int i = 0; i < ogBuffer.size(); i++)
	{
		newBuffer[i] = ogBuffer[i];
	}
}

void checkBuffer(vector<bool> & bitBuffer, vector<bool> & encoding, ofstream &fout)
{
	while (encoding.size() != 0 )
	{
			int availableSpace = 8 - bitBuffer.size();
			for (int i = 0; i < availableSpace; i++)
			{
				if (!encoding.empty())
				{
					bitBuffer.push_back(encoding[0]);
					encoding.erase(encoding.begin());
				}
			}

			if (bitBuffer.size() == 8)
			{
				//convert vector<bool> to bitbuffer for output
				bitset<8> outputBuffer;
				convertBuffer(bitBuffer, outputBuffer);
				unsigned char n = outputBuffer.to_ulong();
				fout << n;
				bitBuffer.clear();
			}

		//cout << "You're Stuck" << endl;
	}
{'fraction_non_alphanumeric': 0.16298958849776896, 'fraction_numerical': 0.015989092711948438, 'mean_word_length': 4.0972836386607705, 'pattern_counts': {'":': 2, '<': 143, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1792425', 'n_tokens_mistral': 3266, 'n_tokens_neox': 2931, 'n_words': 653}