<commit_before>#ifdef _MSC_VER
//fixes "fatal error C1189: #error :  WinSock.h has already been included"
#	include <boost/asio.hpp>
#endif
#include <server/scan_directory.hpp>
#include <server/directory_listing.hpp>
#include <server/sha256.hpp>
#include <server/hexadecimal.hpp>
#include <server/path.hpp>
#include <silicium/asio/tcp_acceptor.hpp>
#include <silicium/observable/coroutine_generator.hpp>
#include <silicium/observable/total_consumer.hpp>
#include <silicium/observable/flatten.hpp>
#include <silicium/asio/writing_observable.hpp>
#include <silicium/asio/reading_observable.hpp>
#include <silicium/source/received_from_socket_source.hpp>
#include <silicium/observable/transform_if_initialized.hpp>
#include <silicium/source/observable_source.hpp>
#include <silicium/observable/for_each.hpp>
#include <silicium/optional.hpp>
#include <silicium/observable/thread.hpp>
#include <silicium/source/file_source.hpp>
#include <silicium/http/http.hpp>
#include <silicium/to_unique.hpp>
#include <silicium/observable/thread_generator.hpp>
#include <silicium/source/buffering_source.hpp>
#include <silicium/open.hpp>
#include <silicium/read_file.hpp>
#include <silicium/source/memory_source.hpp>
#include <silicium/std_threading.hpp>
#include <silicium/source/virtualized_source.hpp>
#include <silicium/source/transforming_source.hpp>
#include <silicium/observable/end.hpp>
#include <silicium/source/single_source.hpp>
#include <silicium/sink/iterator_sink.hpp>
#include <boost/interprocess/sync/null_mutex.hpp>
#include <boost/unordered_map.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/program_options.hpp>
#include <boost/container/vector.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/thread/mutex.hpp>

namespace fileserver
{
	using response_part = Si::memory_range;

	struct content_request
	{
		unknown_digest requested_file;
	};

	template <class String>
	Si::optional<content_request> parse_request_path(String const &path)
	{
		if (path.empty())
		{
			return Si::none;
		}
		auto digest_begin = path.begin();
		if (*digest_begin == '/')
		{
			++digest_begin;
		}
		content_request request;
		auto digest = parse_digest(digest_begin, path.end());
		if (!digest)
		{
			return Si::none;
		}
		request.requested_file = std::move(*digest);
		return std::move(request);
	}

	Si::http::response make_not_found_response()
	{
		Si::http::response header;
		header.http_version = "HTTP/1.0";
		header.status = 404;
		header.status_text = "Not Found";
		(*header.arguments)["Connection"] = "close";
		return header;
	}

	std::vector<char> serialize_response(Si::http::response const &header)
	{
		std::vector<char> serialized;
		auto sink = Si::make_container_sink(serialized);
		Si::http::generate_response(sink, header);<commit_msg>fix a nullptr deref on a 404 access<commit_after>#ifdef _MSC_VER
//fixes "fatal error C1189: #error :  WinSock.h has already been included"
#	include <boost/asio.hpp>
#endif
#include <server/scan_directory.hpp>
#include <server/directory_listing.hpp>
#include <server/sha256.hpp>
#include <server/hexadecimal.hpp>
#include <server/path.hpp>
#include <silicium/asio/tcp_acceptor.hpp>
#include <silicium/observable/coroutine_generator.hpp>
#include <silicium/observable/total_consumer.hpp>
#include <silicium/observable/flatten.hpp>
#include <silicium/asio/writing_observable.hpp>
#include <silicium/asio/reading_observable.hpp>
#include <silicium/source/received_from_socket_source.hpp>
#include <silicium/observable/transform_if_initialized.hpp>
#include <silicium/source/observable_source.hpp>
#include <silicium/observable/for_each.hpp>
#include <silicium/optional.hpp>
#include <silicium/observable/thread.hpp>
#include <silicium/source/file_source.hpp>
#include <silicium/http/http.hpp>
#include <silicium/to_unique.hpp>
#include <silicium/observable/thread_generator.hpp>
#include <silicium/source/buffering_source.hpp>
#include <silicium/open.hpp>
#include <silicium/read_file.hpp>
#include <silicium/source/memory_source.hpp>
#include <silicium/std_threading.hpp>
#include <silicium/source/virtualized_source.hpp>
#include <silicium/source/transforming_source.hpp>
#include <silicium/observable/end.hpp>
#include <silicium/source/single_source.hpp>
#include <silicium/sink/iterator_sink.hpp>
#include <boost/interprocess/sync/null_mutex.hpp>
#include <boost/unordered_map.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/program_options.hpp>
#include <boost/container/vector.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/thread/mutex.hpp>

namespace fileserver
{
	using response_part = Si::memory_range;

	struct content_request
	{
		unknown_digest requested_file;
	};

	template <class String>
	Si::optional<content_request> parse_request_path(String const &path)
	{
		if (path.empty())
		{
			return Si::none;
		}
		auto digest_begin = path.begin();
		if (*digest_begin == '/')
		{
			++digest_begin;
		}
		content_request request;
		auto digest = parse_digest(digest_begin, path.end());
		if (!digest)
		{
			return Si::none;
		}
		request.requested_file = std::move(*digest);
		return std::move(request);
	}

	Si::http::response make_not_found_response()
	{
		Si::http::response header;
		header.http_version = "HTTP/1.0";
		header.status = 404;
		header.status_text = "Not Found";
		header.arguments = Si::make_unique<Si::http::response::arguments_table>();
		(*header.arguments)["Connection"] = "close";
		return header;
	}

	std::vector<char> serialize_response(Si::http::response const &header)
	{
		std::vector<char> serialized;
		auto sink = Si::make_container_sink(serialized);
		Si::http::generate_response(sink, header);
{'fraction_non_alphanumeric': 0.1391196747392611, 'fraction_numerical': 0.004772847799186848, 'mean_word_length': 6.329015544041451, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442575', 'n_tokens_mistral': 2099, 'n_tokens_neox': 1978, 'n_words': 322}