<commit_before>/*
*
*	Author: <NAME>, https://zschoche.org
*
*/
#ifndef __FILESYSTEM_HPP__
#define __FILESYSTEM_HPP__

#include "device_io.hpp"
#include <array>
namespace ext2 {

/*
 * implements the device concept
 */
template <typename Filesystem> class inode : public inode_base<typename Filesystem::device_type> {
      public:
	typedef Filesystem fs_type;

      private:
	fs_type *fs;

	uint32_t get_block_id(uint32_t block_index) const {
		if (block_index < 12) {
			// direct pointer
			return this->data.block_pointer_direct[block_index];
		} else if (block_index < 268) {
			block_index -= 12;
			uint32_t result;
			auto blockid = this->data.block_pointer_indirect[0];
			detail::read_from_device(*fs->device(), fs->to_address(blockid, block_index * sizeof(uint32_t)), result);
			return result;
		} else {
			throw "this is not ready yet";
			return 0; // TODO: implement
		}
	}

      public:
	inode(fs_type *fs, uint64_t offset) : inode_base<typename Filesystem::device_type>(fs->device(), offset), fs(fs) {}

	inline bool is_directory() const { return detail::has_flag(this->data.type, detail::directory); }
	inline bool is_regular_file() const { return detail::has_flag(this->data.type, detail::regular_file); }
	inline bool is_symbolic_link() const { return detail::has_flag(this->data.type, detail::symbolic_link); }
	inline uint64_t size() const {
		if (is_regular_file() && fs->large_files()) {
			return (static_cast<uint64_t>(this->data.dir_acl) << 32) | (this->data.size);
		} else {
			return this->data.size;
		}
	}

	void read(uint64_t offset, char *buffer, uint64_t length) {
		auto buffer_offset = 0;
		do {
			auto block_index = offset / fs->block_size();
			auto block_offset = offset % fs->block_size();
			auto block_length = std::min(fs->block_size() - block_offset, length);
			fs->device()->read(fs->to_address(get_block_id(block_index), block_offset), &buffer[buffer_offset], block_length);
			buffer_offset += block_length;
			offset += block_length;
			length -= block_length;
		} while (length > 0);
	}
	void write(uint64_t offset, const char *buffer, uint64_t length) {} // TODO: implement!

	inline fs_type *get_fs() { return fs; }
};
template <typename OStream, typename Inode> void read_inode_content(OStream &os, Inode& inode) {
		std::array<char, 255> buffer;
		auto offset = 0;
		const auto size = inode.size();
		do {
			auto length = std::min<uint64_t>(size - offset, buffer.size());
			inode.read(offset, buffer.data(), length);
			os << std::string(buffer.data(), length);
			offset += length;
		} while (offset < size);
	}


typedef std::vector<detail::directory_entry> directory_entry_list;

namespace inodes {
template <typename Filesystem> struct file : inode<Filesystem> {

	template <typename OStream> void read_full_file(OStream &os) {
		read_inode_content(os, *this);
	}
};
template <typename OStream, typename Filesystem> OStream &operator<<(OStream &os, file<Filesystem> &f) {
	f.read_full_file(os);
	return os;
}

template <typename Filesystem> struct character_device : inode<Filesystem> {};
template <typename Filesystem> struct block_device : inode<Filesystem> {};
template <typename Filesystem> struct fifo : inode<Filesystem> {};
template <typename Filesystem> struct symbolic_link : inode<Filesystem> {

	std::string get_target() {
		/* http://www.nongnu.org/ext2-doc/ext2.html#DEF-SYMBOLIC-LINKS */
		if(this->size() < 60) {
			const char* cstring = reinterpret_cast<char*>(&(this->data.block_pointer_direct[0]));
			std::string result(cstring, this->size());
			return result;
		}

		std::stringstream ss;
		read_inode_content(ss, *this);
		return ss.str();
	}
};


template <typename Filesystem> struct directory : inode<Filesystem> {

	directory_entry_list read_entrys() {
		auto offset = 0;
		directory_entry_list result;
		result.reserve(8);
		do {
			detail::directory_entry entry;
			detail::read_from_device(*this, offset, entry, 8);
			if (entry.inode_id == 0)
				break;
			offset += 8;
			entry.name.resize(entry.name_size);
			this->read(offset, const_cast<char *>(entry.name.c_str()), entry.name_size);
			offset += entry.size - 8;
			result.push_back(std::move(entry));
		} while (offset < this->size());
		return result;
	}

	void write_entrys(const directory_entry_list &entrys) {
		// TODO
	}
};
} /* namespace inodes */

template <typename Filesystem> inodes::directory<Filesystem> *to_directory(inode<Filesystem> *from) {
	if (from->is_directory())
		return static_cast<inodes::directory<Filesystem> *>(from);
	else
		return nullptr;
}
template <typename Filesystem> inodes::file<Filesystem> *to_file(inode<Filesystem> *from) {
	if (from->is_regular_file())
		return static_cast<inodes::file<Filesystem> *>(from);
	else
		return nullptr;
}

template <typename Filesystem> inodes::symbolic_link<Filesystem> *to_symbolic_link(inode<Filesystem> *from) {
	if (from->is_symbolic_link())
		return static_cast<inodes::symbolic_link<Filesystem> *>(from);
	else
		return nullptr;
}

typedef std::vector<std::pair<uint64_t, std::string *> > path;

template <typename OStream> OStream &operator<<(OStream &os, const path &p) {
	for (const auto &item : p) {
		os << '/' << *item.second;
	}
	return os;
}
<commit_msg>Added a type for paths<commit_after>/*
*
*	Author: <NAME>, https://zschoche.org
*
*/
#ifndef __FILESYSTEM_HPP__
#define __FILESYSTEM_HPP__

#include "device_io.hpp"
#include <array>
#include <boost/algorithm/string/split.hpp>
namespace ext2 {

/*
 * implements the device concept
 */
template <typename Filesystem> class inode : public inode_base<typename Filesystem::device_type> {
      public:
	typedef Filesystem fs_type;

      private:
	fs_type *fs;

	uint32_t get_block_id(uint32_t block_index) const {
		if (block_index < 12) {
			// direct pointer
			return this->data.block_pointer_direct[block_index];
		} else if (block_index < 268) {
			block_index -= 12;
			uint32_t result;
			auto blockid = this->data.block_pointer_indirect[0];
			detail::read_from_device(*fs->device(), fs->to_address(blockid, block_index * sizeof(uint32_t)), result);
			return result;
		} else {
			throw "this is not ready yet";
			return 0; // TODO: implement
		}
	}

      public:
	inode(fs_type *fs, uint64_t offset) : inode_base<typename Filesystem::device_type>(fs->device(), offset), fs(fs) {}

	inline bool is_directory() const { return detail::has_flag(this->data.type, detail::directory); }
	inline bool is_regular_file() const { return detail::has_flag(this->data.type, detail::regular_file); }
	inline bool is_symbolic_link() const { return detail::has_flag(this->data.type, detail::symbolic_link); }
	inline uint64_t size() const {
		if (is_regular_file() && fs->large_files()) {
			return (static_cast<uint64_t>(this->data.dir_acl) << 32) | (this->data.size);
		} else {
			return this->data.size;
		}
	}

	void read(uint64_t offset, char *buffer, uint64_t length) {
		auto buffer_offset = 0;
		do {
			auto block_index = offset / fs->block_size();
			auto block_offset = offset % fs->block_size();
			auto block_length = std::min(fs->block_size() - block_offset, length);
			fs->device()->read(fs->to_address(get_block_id(block_index), block_offset), &buffer[buffer_offset], block_length);
			buffer_offset += block_length;
			offset += block_length;
			length -= block_length;
		} while (length > 0);
	}
	void write(uint64_t offset, const char *buffer, uint64_t length) {} // TODO: implement!

	inline fs_type *get_fs() { return fs; }
};
template <typename OStream, typename Inode> void read_inode_content(OStream &os, Inode &inode) {
	std::array<char, 255> buffer;
	auto offset = 0;
	const auto size = inode.size();
	do {
		auto length = std::min<uint64_t>(size - offset, buffer.size());
		inode.read(offset, buffer.data(), length);
		os << std::string(buffer.data(), length);
		offset += length;
	} while (offset < size);
}

typedef std::vector<detail::directory_entry> directory_entry_list;

namespace inodes {
template <typename Filesystem> struct file : inode<Filesystem> {

	template <typename OStream> void read_full_file(OStream &os) { read_inode_content(os, *this); }
};
template <typename OStream, typename Filesystem> OStream &operator<<(OStream &os, file<Filesystem> &f) {
	f.read_full_file(os);
	return os;
}

template <typename Filesystem> struct character_device : inode<Filesystem> {};
template <typename Filesystem> struct block_device : inode<Filesystem> {};
template <typename Filesystem> struct fifo : inode<Filesystem> {};
template <typename Filesystem> struct symbolic_link : inode<Filesystem> {

	std::string get_target() {
		/* http://www.nongnu.org/ext2-doc/ext2.html#DEF-SYMBOLIC-LINKS */
		if (this->size() < 60) {
			const char *cstring = reinterpret_cast<char *>(&(this->data.block_pointer_direct[0]));
			std::string result(cstring, this->size());
			return result;
		}

		std::stringstream ss;
		read_inode_content(ss, *this);
		return ss.str();
	}
};

template <typename Filesystem> struct directory : inode<Filesystem> {

	directory_entry_list read_entrys() {
		auto offset = 0;
		directory_entry_list result;
		result.reserve(8);
		do {
			detail::directory_entry entry;
			detail::read_from_device(*this, offset, entry, 8);
			if (entry.inode_id == 0)
				break;
			offset += 8;
			entry.name.resize(entry.name_size);
			this->read(offset, const_cast<char *>(entry.name.c_str()), entry.name_size);
			offset += entry.size - 8;
			result.push_back(std::move(entry));
		} while (offset < this->size());
		return result;
	}

	void write_entrys(const directory_entry_list &entrys) {
		// TODO
	}
};
} /* namespace inodes */

template <typename Filesystem> inodes::directory<Filesystem> *to_directory(inode<Filesystem> *from) {
	if (from->is_directory())
		return static_cast<inodes::directory<Filesystem> *>(from);
	else
		return nullptr;
}
template <typename Filesystem> inodes::file<Filesystem> *to_file(inode<Filesystem> *from) {
	if (from->is_regular_file())
		return static_cast<inodes::file<Filesystem> *>(from);
	else
		return nullptr;
}

template <typename Filesystem> inodes::symbolic_link<Filesystem> *to_symbolic_link(inode<Filesystem> *from) {
	if (from->is_symbolic_link())
		return static_cast<inodes::symbolic_link<Filesystem> *>(from);
	else
		return nullptr;
}

struct path {
	// typedef boost::iterator_range<std::string::iterator> range_type;
	typedef std::vector<std::string> vec_type;
	std::string str;
	vec_type vec;

	inline bool is_relative() const { return !(!str.empty() && str[0] == '/'); }
};

inline path path_from_string(const std::string &p) {
	path result{p};
	if (result.str.size() > 3) {
		auto first = result.str.begin();
		auto last = --(result.str.end());
		if (*first == '/')
			first++;
		if (*last != '/')
			last++;
		auto range = boost::make_iterator_range(first, last);
		boost::split(result.vec, range, [](auto c) { return c == '/'; });
	}
	return result;
}

{'fraction_non_alphanumeric': 0.14833287152489147, 'fraction_numerical': 0.010159785720882978, 'mean_word_length': 4.955995599559956, 'pattern_counts': {'":': 0, '<': 121, '<?xml version=': 0, '>': 148, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3185635', 'n_tokens_mistral': 3953, 'n_tokens_neox': 3641, 'n_words': 827}