<commit_before>#include "archive.hpp"
#include "../huffman/compress.hpp"
#include "../binary/bin_stream.hpp"

#include<fstream>
#include<vector>
#include<experimental/filesystem>

void archive_files(const std::vector<std::string>& filenames, std::ostream& out_stream) {
	std::ostreambuf_iterator<char> out_iterator(out_stream);

	for(auto& fn : filenames) { // TODO: must strip base path
		for(char c : fn) *out_iterator = c;
		*out_iterator = '\0';
	}
	*out_iterator = '\0';

	BinaryWrite bin_out(out_iterator);

	for(auto& fn : filenames) {
		std::ifstream in(fn);
		auto freq_table = readTable(in);
		in.close();
		in.open(fn);
		compress(freq_table, in, bin_out);
	}
}

void archive(const char* in_file, const char* out_file) {
	std::ofstream out(out_file);
	archive(in_file, out);
}

void archive(const char* in_file, std::ostream& out_stream) {
	namespace fs = std::experimental::filesystem;

	std::vector<std::string> filenames;

	if(fs::is_regular_file(in_file)) {
		filenames.push_back(in_file);
	} else if(fs::is_directory(in_file)) {
		for(auto& p: fs::recursive_directory_iterator(in_file)) {
			if(!fs::is_regular_file(p)) continue;
			filenames.push_back(p.path());
		}
	} // else not supported, yet!

	archive_files(filenames, out_stream);
}

void archive(std::istream& in_stream, std::ostream& out_stream) {
	std::vector<std::string> filenames;
	std::string fn;

	while(std::getline(in_stream, fn))
		filenames.push_back(fn);

	archive_files(filenames, out_stream);
}
<commit_msg>Strip base path on archiving<commit_after>#include "archive.hpp"
#include "../huffman/compress.hpp"
#include "../binary/bin_stream.hpp"

#include<fstream>
#include<vector>
#include<experimental/filesystem>

void archive_files(const std::vector<std::string>& filenames, std::ostream& out_stream) {
	std::ostreambuf_iterator<char> out_iterator(out_stream);

	for(auto& fn : filenames) {
		for(char c : fn) *out_iterator = c;
		*out_iterator = '\0';
	}
	*out_iterator = '\0';

	BinaryWrite bin_out(out_iterator);

	for(auto& fn : filenames) {
		std::ifstream in(fn);
		auto freq_table = readTable(in);
		in.close();
		in.open(fn);
		compress(freq_table, in, bin_out);
	}
}

void archive(const char* in_file, const char* out_file) {
	std::ofstream out(out_file);
	archive(in_file, out);
}

void archive(const char* in_file, std::ostream& out_stream) {
	namespace fs = std::experimental::filesystem;

	std::vector<std::string> filenames;

	auto parent_path = fs::path(in_file).parent_path();
	if(!parent_path.empty())
		fs::current_path(fs::path(in_file).parent_path());

	auto file = fs::path(in_file).filename();

	if(fs::is_regular_file(file)) {
		filenames.push_back(file);
	} else if(fs::is_directory(file)) {
		for(auto& p: fs::recursive_directory_iterator(file)) {
			if(!fs::is_regular_file(p)) continue;
			filenames.push_back(p.path());
		}
	} // else not supported, yet!

	archive_files(filenames, out_stream);
}

void archive(std::istream& in_stream, std::ostream& out_stream) {
	std::vector<std::string> filenames;
	std::string fn;

	while(std::getline(in_stream, fn))
		filenames.push_back(fn);

	archive_files(filenames, out_stream);
}

{'fraction_non_alphanumeric': 0.16327180140038192, 'fraction_numerical': 0.001273074474856779, 'mean_word_length': 6.078828828828829, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3700787', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1175, 'n_words': 209}