<commit_before>/*

Copyright (c) 2003, <NAME>, <NAME>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "libtorrent/pch.hpp"

#include <ctime>
#include <iterator>
#include <algorithm>
#include <set>
#include <functional>

#ifdef _MSC_VER
#pragma warning(push, 1)
#endif

#include <boost/lexical_cast.hpp>
#include <boost/filesystem/convenience.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/ref.hpp>
#include <boost/bind.hpp>
#include <boost/version.hpp>
#include <boost/scoped_array.hpp>
#if BOOST_VERSION >= 103500
#include <boost/system/system_error.hpp>
#endif

#ifdef _MSC_VER
#pragma warning(pop)
#endif

#include "libtorrent/config.hpp"
#include "libtorrent/storage.hpp"
#include "libtorrent/torrent.hpp"
#include "libtorrent/hasher.hpp"
#include "libtorrent/session.hpp"
#include "libtorrent/peer_id.hpp"
#include "libtorrent/file.hpp"
#include "libtorrent/invariant_check.hpp"
#include "libtorrent/file_pool.hpp"
#include "libtorrent/aux_/session_impl.hpp"
#include "libtorrent/disk_buffer_holder.hpp"
#include "libtorrent/alloca.hpp"

#include <cstdio>

//#define TORRENT_PARTIAL_HASH_LOG

#if TORRENT_USE_IOSTREAM
#include <boost/filesystem/fstream.hpp>
#include <ios>
#include <iostream>
#include <iomanip>
#endif

#if defined(__APPLE__)
// for getattrlist()
#include <sys/attr.h>
#include <unistd.h>
// for statfs()
#include <sys/param.h>
#include <sys/mount.h>
#endif

#if defined(__linux__)
#include <sys/statfs.h>
#endif

#if defined(__FreeBSD__)
// for statfs()
#include <sys/param.h>
#include <sys/mount.h>
#endif

#if TORRENT_USE_WPATH || TORRENT_USE_LOCALE_FILENAMES
// for convert_to_wstring and convert_to_native
#include "libtorrent/escape_string.hpp"
#endif

namespace fs = boost::filesystem;
using boost::bind;
using namespace ::boost::multi_index;
using boost::multi_index::multi_index_container;

#if defined TORRENT_DEBUG && defined TORRENT_STORAGE_DEBUG && TORRENT_USE_IOSTREAM
namespace
{
	using namespace libtorrent;

	void print_to_log(const std::string& s)
	{
		static std::ofstream log("log.txt");
		log << s;
		log.flush();
	}
}
#endif

namespace libtorrent
{
	template <class Path>
	void recursive_copy(Path const& old_path, Path const& new_path, error_code& ec)
	{
		using boost::filesystem::basic_directory_iterator;
#ifndef BOOST_NO_EXCEPTIONS
		try {
#endif
		TORRENT_ASSERT(!ec);
		if (is_directory(old_path))
		{
			create_directory(new_path);
			for (basic_directory_iterator<Path> i(old_path), end; i != end; ++i)
			{
#if BOOST_VERSION < 103600
				recursive_copy(i->path(), new_path / i->path().leaf(), ec);
#else
				recursive_copy(i->path(), new_path / i->path().filename(), ec);
#endif
				if (ec) return;
			}
		}
		else
		{
			copy_file(old_path, new_path);
		}
#ifndef BOOST_NO_EXCEPTIONS
		}
#if BOOST_VERSION >= 103500
		catch (boost::system::system_error& e)
		{
			ec = e.code();
		}
#else
		catch (boost::filesystem::filesystem_error& e)
		{
			ec = error_code(e.system_error(), get_system_category());
		}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
	}

	template <class Path>
	void recursive_remove(Path const& old_path)
	{
		using boost::filesystem::basic_directory_iterator;
#ifndef BOOST_NO_EXCEPTIONS
		try {
#endif
		if (is_directory(old_path))
		{
			for (basic_directory_iterator<Path> i(old_path), end; i != end; ++i)
				recursive_remove(i->path());
			remove(old_path);
		}
		else
		{
			remove(old_path);
		}
#ifndef BOOST_NO_EXCEPTIONS
		} catch (std::exception& e) {}
#endif
	}
	std::vector<std::pair<size_type, std::time_t> > get_filesizes(
		file_storage const& s, fs::path p)
	{
		p = complete(p);
		std::vector<std::pair<size_type, std::time_t> > sizes;
		for (file_storage::iterator i = s.begin()
			, end(s.end());i != end; ++i)
		{
			size_type size = 0;
			std::time_t time = 0;
			if (i->pad_file)
			{
				sizes.push_back(std::make_pair(i->size, time));
				continue;
			}
#if TORRENT_USE_WPATH
			fs::wpath f = convert_to_wstring((p / i->path).string());
#elif TORRENT_USE_LOCALE_FILENAMES
			fs::path f = convert_to_native((p / i->path).string());
#else
			fs::path f = p / i->path;
#endif
			// TODO: optimize
			if (exists(f))
#ifndef BOOST_NO_EXCEPTIONS
			try
#endif
			{
				size = file_size(f);
				time = last_write_time(f);
			}
#ifndef BOOST_NO_EXCEPTIONS
			catch (std::exception&) {}
#endif
			sizes.push_back(std::make_pair(size, time));
		}
		return sizes;
	}

	// matches the sizes and timestamps of the files passed in
	// in non-compact mode, actual file sizes and timestamps
	// are allowed to be bigger and more recent than the fast
	// resume data. This is because full allocation will not move
	// pieces, so any older version of the resume data will
	// still be a correct subset of the actual data on disk.
	bool match_filesizes(
		file_storage const& fs
		, fs::path p
		, std::vector<std::pair<size_type, std::time_t> > const& sizes
		, bool compact_mode
		, std::string* error)
	{
		if ((int)sizes.size() != fs.num_files())
		{
			if (error) *error = "mismatching number of files";
			return false;
		}
		p = complete(p);

		std::vector<std::pair<size_type, std::time_t> >::const_iterator s
			= sizes.begin();
		for (file_storage::iterator i = fs.begin()
			, end(fs.end());i != end; ++i, ++s)
		{
			size_type size = 0;
			std::time_t time = 0;
			if (i->pad_file) continue;

#if TORRENT_USE_WPATH
			fs::wpath f = convert_to_wstring((p / i->path).string());
#elif TORRENT_USE_LOCALE_FILENAMES
			fs::path f = convert_to_native((p / i->path).string());
#else
			fs::path f = p / i->path;
#endif
			// TODO: Optimize this! This will result in 3 stat calls per file!
			if (exists(f))
#ifndef BOOST_NO_EXCEPTIONS
			try
#endif
			{
				size = file_size(f);
				time = last_write_time(f);
			}
#ifndef BOOST_NO_EXCEPTIONS
			catch (std::exception&) {}
#endif
			if ((compact_mode && size != s->first)
				|| (!compact_mode && size < s->first))
			{
				if (error) *error = "filesize mismatch for file '"
					+ i->path.external_file_string()
					+ "', size: " + boost::lexical_cast<std::string>(size)
					+ ", expected to be " + boost::lexical_cast<std::string>(s->first)
					+ " bytes";
				return false;
			}
			// allow one second 'slack', because of FAT volumes
			// in sparse mode, allow the files to be more recent
			// than the resume data, but only by 5 minutes
			if ((compact_mode && (time > s->second + 1 || time < s->second - 1)) ||
				(!compact_mode && (time > s->second + 5 * 60 || time < s->second - 1)))
			{
				if (error) *error = "timestamp mismatch for file '"
					+ i->path.external_file_string()
					+ "', modification date: " + boost::lexical_cast<std::string>(time)
					+ ", expected to have modification date "
					+ boost::lexical_cast<std::string>(s->second);
				return false;
			}
		}
		return true;
	}

	// for backwards compatibility, let the default readv and
	// writev implementations be implemented in terms of the
	// old read and write
	int storage_interface::readv(file::iovec_t const* bufs
		, int slot, int offset, int num_bufs)
	{
		int ret = 0;
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
		{
			int r = write((char const*)i->iov_base, slot, offset, i->iov_len);
			offset += i->iov_len;
			if (r == -1) return -1;
			ret += r;
		}
		return ret;
	}

	int storage_interface::writev(file::iovec_t const* bufs, int slot
		, int offset, int num_bufs)
	{
		int ret = 0;
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
		{
			int r = read((char*)i->iov_base, slot, offset, i->iov_len);
			offset += i->iov_len;
			if (r == -1) return -1;
			ret += r;
		}
		return ret;
	}

	int copy_bufs(file::iovec_t const* bufs, int bytes, file::iovec_t* target)
	{
		int size = 0;
		int ret = 1;
		for (;;)
		{
			*target = *bufs;
			size += bufs->iov_len;
			if (size >= bytes)
			{
				target->iov_len -= size - bytes;
				return ret;
			}
			++bufs;
			++target;
			++ret;
		}
	}

	void advance_bufs(file::iovec_t*& bufs, int bytes)
	{
		int size = 0;
		for (;;)
		{
			size += bufs->iov_len;
			if (size >= bytes)
			{
				((char*&)bufs->iov_base) += bufs->iov_len - (size - bytes);
				bufs->iov_len = size - bytes;
				return;
			}
			++bufs;
		}
	}

	int bufs_size(file::iovec_t const* bufs, int num_bufs)
	{
		int size = 0;
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
			size += i->iov_len;
		return size;
	}
	
	void clear_bufs(file::iovec_t const* bufs, int num_bufs)
	{
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
			std::memset(i->iov_base, 0, i->iov_len);
	}

#ifdef TORRENT_DEBUG
	int count_bufs(file::iovec_t const* bufs, int bytes)
	{
		int size = 0;
		int count = 1;
		if (bytes == 0) return 0;
		for (file::iovec_t const* i = bufs;; ++i, ++count)
		{
			size += i->iov_len;
			TORRENT_ASSERT(size <= bytes);
			if (size >= bytes) return count;
		}
	}
#endif

	class storage : public storage_interface, boost::noncopyable
	{
	public:
		storage(file_storage const& fs, fs::path const& path, file_pool& fp)
			: m_files(fs)
			, m_pool(fp)
			, m_page_size(4096)
			, m_allocate_files(false)
		{
			TORRENT_ASSERT(m_files.begin() != m_files.end());
			m_save_path = fs::complete(path);
			TORRENT_ASSERT(m_save_path.is_complete());
#ifdef TORRENT_WINDOWS
			SYSTEM_INFO si;
			GetSystemInfo(&si);
			m_page_size = si.dwPageSize;
#else
			m_page_size = sysconf(_SC_PAGESIZE);
#endif
		}

		bool has_any_file();
		bool rename_file(int index, std::string const& new_filename);
		bool release_files();
		bool delete_files();
		bool initialize(bool allocate_files);
		bool move_storage(fs::path save_path);
		int read(char* buf, int slot, int offset, int size);
		int write(char const* buf, int slot, int offset, int size);
		int sparse_end(int start) const;
		int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs);
		int writev(file::iovec_t const* buf, int slot, int offset, int num_bufs);
		bool move_slot(int src_slot, int dst_slot);
		bool swap_slots(int slot1, int slot2);
		bool swap_slots3(int slot1, int slot2, int slot3);
		bool verify_resume_data(lazy_entry const& rd, std::string& error);
		bool write_resume_data(entry& rd) const;
		sha1_hash hash_for_slot(int slot, partial_hash& ph, int piece_size);

		// this identifies a read or write operation
		// so that storage::readwrite() knows what to
		// do when it's actually touching the file
		struct fileop
		{
			size_type (file::*regular_op)(size_type file_offset
				, file::iovec_t const* bufs, int num_bufs, error_code& ec);
			size_type (storage::*unaligned_op)(boost::shared_ptr<file> const& f
				, size_type file_offset, file::iovec_t const* bufs, int num_bufs
				, error_code& ec);
			int cache_setting;
			int mode;
		};

		void delete_one_file(std::string const& p);
		int readwritev(file::iovec_t const* bufs, int slot, int offset
			, int num_bufs, fileop const&);

		~storage()
		{ m_pool.release(this); }

		size_type read_unaligned(boost::shared_ptr<file> const& file_handle
			, size_type file_offset, file::iovec_t const* bufs, int num_bufs, error_code& ec);
		size_type write_unaligned(boost::shared_ptr<file> const& file_handle
			, size_type file_offset, file::iovec_t const* bufs, int num_bufs, error_code& ec);

		file_storage const& files() const { return m_mapped_files?*m_mapped_files:m_files; }

		boost::scoped_ptr<file_storage> m_mapped_files;
		file_storage const& m_files;

		std::vector<boost::uint8_t> m_file_priority;
		fs::path m_save_path;
		// the file pool is typically stored in
		// the session, to make all storage
		// instances use the same pool
		file_pool& m_pool;

		int m_page_size;
		bool m_allocate_files;
	};

	sha1_hash storage::hash_for_slot(int slot, partial_hash& ph, int piece_size)
	{
		TORRENT_ASSERT(!error());
		int slot_size = piece_size - ph.offset;
		if (slot_size > 0)
		{
			int block_size = 16 * 1024;
			if (disk_pool()) block_size = disk_pool()->block_size();
			int size = slot_size;
			int num_blocks = (size + block_size - 1) / block_size;
			file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks);
			for (int i = 0; i < num_blocks; ++i)
			{
				bufs[i].iov_base = disk_pool()->allocate_buffer("hash temp");
				bufs[i].iov_len = (std::min)(block_size, size);
				size -= bufs[i].iov_len;
			}
			readv(bufs, slot, ph.offset, num_blocks);

			for (int i = 0; i < num_blocks; ++i)
			{
				ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len);
				disk_pool()->free_buffer((char*)bufs[i].iov_base);
			}
			if (error()) return sha1_hash(0);
		}
		return ph.h.final();
	}

	bool storage::initialize(bool allocate_files)
	{
		m_allocate_files = allocate_files;
		error_code ec;
		// first, create all missing directories
		fs::path last_path;
		for (file_storage::iterator file_iter = files().begin(),
			end_iter = files().end(); file_iter != end_iter; ++file_iter)
		{
			fs::path dir = (m_save_path / file_iter->path).branch_path();

			if (dir != last_path)
			{
				last_path = dir;

#if TORRENT_USE_WPATH
				fs::wpath wp = convert_to_wstring(last_path.string());
				if (!exists(wp))
					create_directories(wp);
#elif TORRENT_USE_LOCALE_FILENAMES
				fs::path p = convert_to_native(last_path.string());
				if (!exists(p))
					create_directories(p);
#else
				if (!exists(last_path))
					create_directories(last_path);
#endif
			}

			int file_index = file_iter - files().begin();

			// ignore files that have priority 0
			if (int(m_file_priority.size()) > file_index
				&& m_file_priority[file_index] == 0) continue;

			// ignore pad files
			if (file_iter->pad_file) continue;

#ifndef BOOST_NO_EXCEPTIONS
			try {
#endif

#if TORRENT_USE_WPATH
			fs::wpath file_path = convert_to_wstring((m_save_path / file_iter->path).string());
#else
			fs::path file_path = m_save_path / file_iter->path;
#endif
			// if the file is empty, just create it either way.
			// if the file already exists, but is larger than what
			// it's supposed to be, also truncate it
			if (allocate_files
				|| file_iter->size == 0
				|| (exists(file_path) && file_size(file_path) > file_iter->size))
			{
				error_code ec;
				int mode = file::read_write;
				if (m_settings
					&& (settings().disk_io_read_mode == session_settings::disable_os_cache
					|| (settings().disk_io_read_mode == session_settings::disable_os_cache_for_aligned_files
					&& ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0)))
					mode |= file::no_buffer;
				if (!m_allocate_files) mode |= file::sparse;
				boost::shared_ptr<file> f = m_pool.open_file(this
					, m_save_path / file_iter->path, mode, ec);
				if (ec) set_error(m_save_path / file_iter->path, ec);
				else if (f)
				{
					f->set_size(file_iter->size, ec);
					if (ec) set_error(m_save_path / file_iter->path, ec);
				}
			}
#ifndef BOOST_NO_EXCEPTIONS
			}
#if BOOST_VERSION >= 103500
			catch (boost::system::system_error& e)
			{
				set_error(m_save_path / file_iter->path, e.code());
				return true;
			}
#else
			catch (boost::filesystem::filesystem_error& e)
			{
				set_error(m_save_path / file_iter->path
					, error_code(e.system_error(), get_system_category()));
				return true;
			}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
		}
		std::vector<boost::uint8_t>().swap(m_file_priority);
		// close files that were opened in write mode
		m_pool.release(this);
		return false;
	}

	bool storage::has_any_file()
	{
		file_storage::iterator i = files().begin();
		file_storage::iterator end = files().end();

		for (; i != end; ++i)
		{
			bool file_exists = false;
#if TORRENT_USE_LOCALE_FILENAMES
			fs::path f = convert_to_native((m_save_path / i->path).string());
#else
			fs::path f = m_save_path / i->path;
#endif
#ifndef BOOST_NO_EXCEPTIONS
			try
			{
#endif
#if TORRENT_USE_WPATH
				fs::wpath wf = convert_to_wstring(f.string());
				file_exists = exists(wf);
#else
				file_exists = exists(f);
#endif
#ifndef BOOST_NO_EXCEPTIONS
			}
#if BOOST_VERSION >= 103500
			catch (boost::system::system_error& e)
			{
				set_error(f, e.code());
				return false;
			}
#else
			catch (boost::filesystem::filesystem_error& e)
			{
				set_error(f, error_code(e.system_error(), get_system_category()));
				return false;
			}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
			if (file_exists && i->size > 0)
				return true;
		}
		return false;
	}

	bool storage::rename_file(int index, std::string const& new_filename)
	{
		if (index < 0 || index >= m_files.num_files()) return true;
		fs::path old_name = m_save_path / files().at(index).path;
		m_pool.release(old_name);

#if TORRENT_USE_WPATH
		fs::wpath old_path = convert_to_wstring(old_name.string());
		fs::wpath new_path = convert_to_wstring((m_save_path / new_filename).string());
#elif TORRENT_USE_LOCALE_FILENAMES
		fs::path const& old_path = convert_to_native(old_name.string());
		fs::path new_path = convert_to_native((m_save_path / new_filename).string());
#else
		fs::path const& old_path = old_name;
		fs::path new_path = m_save_path / new_filename;
#endif

#ifndef BOOST_NO_EXCEPTIONS
		try
		{
#endif
			// if old path doesn't exist, just rename the file
			// in our file_storage, so that when it is created
			// it will get the new name
			create_directories(new_path.branch_path());
			if (exists(old_path)) rename(old_path, new_path);
/*
			error_code ec;
			rename(old_path, new_path, ec);
			if (ec)
			{
				set_error(old_path, ec);
				return;
			}
*/
			if (!m_mapped_files)
			{ m_mapped_files.reset(new file_storage(m_files)); }
			m_mapped_files->rename_file(index, new_filename);
#ifndef BOOST_NO_EXCEPTIONS
		}
#if BOOST_VERSION >= 103500
		catch (boost::system::system_error& e)
		{
			set_error(old_name, e.code());
			return true;
		}
#else
		catch (boost::filesystem::filesystem_error& e)
		{
			set_error(old_name, error_code(e.system_error()
				, get_system_category()));
			return true;
		}
#endif // BOOST_VERSION
#endif
		return false;
	}

	bool storage::release_files()
	{
		m_pool.release(this);
		return false;
	}

	void storage::delete_one_file(std::string const& p)
	{
#if TORRENT_USE_WPATH
#ifndef BOOST_NO_EXCEPTIONS
		try
#endif
		{ fs::remove(safe_convert(p)); }
#ifndef BOOST_NO_EXCEPTIONS
#if BOOST_VERSION >= 103500
		catch (boost::system::system_error& e)
		{
			set_error(p, e.code());
		}
#else
		catch (boost::filesystem::filesystem_error& e)
		{
			set_error(p, error_code(e.system_error(), get_system_category()));
		}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
#elif TORRENT_USE_LOCALE_FILENAMES
		if (std::remove(convert_to_native(p).c_str()) != 0 && errno != ENOENT)<commit_msg>fixed missed spot in safe_convert -> convert_to_wstring rename<commit_after>/*

Copyright (c) 2003, <NAME>, <NAME>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

*/

#include "libtorrent/pch.hpp"

#include <ctime>
#include <iterator>
#include <algorithm>
#include <set>
#include <functional>

#ifdef _MSC_VER
#pragma warning(push, 1)
#endif

#include <boost/lexical_cast.hpp>
#include <boost/filesystem/convenience.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/ref.hpp>
#include <boost/bind.hpp>
#include <boost/version.hpp>
#include <boost/scoped_array.hpp>
#if BOOST_VERSION >= 103500
#include <boost/system/system_error.hpp>
#endif

#ifdef _MSC_VER
#pragma warning(pop)
#endif

#include "libtorrent/config.hpp"
#include "libtorrent/storage.hpp"
#include "libtorrent/torrent.hpp"
#include "libtorrent/hasher.hpp"
#include "libtorrent/session.hpp"
#include "libtorrent/peer_id.hpp"
#include "libtorrent/file.hpp"
#include "libtorrent/invariant_check.hpp"
#include "libtorrent/file_pool.hpp"
#include "libtorrent/aux_/session_impl.hpp"
#include "libtorrent/disk_buffer_holder.hpp"
#include "libtorrent/alloca.hpp"

#include <cstdio>

//#define TORRENT_PARTIAL_HASH_LOG

#if TORRENT_USE_IOSTREAM
#include <boost/filesystem/fstream.hpp>
#include <ios>
#include <iostream>
#include <iomanip>
#endif

#if defined(__APPLE__)
// for getattrlist()
#include <sys/attr.h>
#include <unistd.h>
// for statfs()
#include <sys/param.h>
#include <sys/mount.h>
#endif

#if defined(__linux__)
#include <sys/statfs.h>
#endif

#if defined(__FreeBSD__)
// for statfs()
#include <sys/param.h>
#include <sys/mount.h>
#endif

#if TORRENT_USE_WPATH || TORRENT_USE_LOCALE_FILENAMES
// for convert_to_wstring and convert_to_native
#include "libtorrent/escape_string.hpp"
#endif

namespace fs = boost::filesystem;
using boost::bind;
using namespace ::boost::multi_index;
using boost::multi_index::multi_index_container;

#if defined TORRENT_DEBUG && defined TORRENT_STORAGE_DEBUG && TORRENT_USE_IOSTREAM
namespace
{
	using namespace libtorrent;

	void print_to_log(const std::string& s)
	{
		static std::ofstream log("log.txt");
		log << s;
		log.flush();
	}
}
#endif

namespace libtorrent
{
	template <class Path>
	void recursive_copy(Path const& old_path, Path const& new_path, error_code& ec)
	{
		using boost::filesystem::basic_directory_iterator;
#ifndef BOOST_NO_EXCEPTIONS
		try {
#endif
		TORRENT_ASSERT(!ec);
		if (is_directory(old_path))
		{
			create_directory(new_path);
			for (basic_directory_iterator<Path> i(old_path), end; i != end; ++i)
			{
#if BOOST_VERSION < 103600
				recursive_copy(i->path(), new_path / i->path().leaf(), ec);
#else
				recursive_copy(i->path(), new_path / i->path().filename(), ec);
#endif
				if (ec) return;
			}
		}
		else
		{
			copy_file(old_path, new_path);
		}
#ifndef BOOST_NO_EXCEPTIONS
		}
#if BOOST_VERSION >= 103500
		catch (boost::system::system_error& e)
		{
			ec = e.code();
		}
#else
		catch (boost::filesystem::filesystem_error& e)
		{
			ec = error_code(e.system_error(), get_system_category());
		}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
	}

	template <class Path>
	void recursive_remove(Path const& old_path)
	{
		using boost::filesystem::basic_directory_iterator;
#ifndef BOOST_NO_EXCEPTIONS
		try {
#endif
		if (is_directory(old_path))
		{
			for (basic_directory_iterator<Path> i(old_path), end; i != end; ++i)
				recursive_remove(i->path());
			remove(old_path);
		}
		else
		{
			remove(old_path);
		}
#ifndef BOOST_NO_EXCEPTIONS
		} catch (std::exception& e) {}
#endif
	}
	std::vector<std::pair<size_type, std::time_t> > get_filesizes(
		file_storage const& s, fs::path p)
	{
		p = complete(p);
		std::vector<std::pair<size_type, std::time_t> > sizes;
		for (file_storage::iterator i = s.begin()
			, end(s.end());i != end; ++i)
		{
			size_type size = 0;
			std::time_t time = 0;
			if (i->pad_file)
			{
				sizes.push_back(std::make_pair(i->size, time));
				continue;
			}
#if TORRENT_USE_WPATH
			fs::wpath f = convert_to_wstring((p / i->path).string());
#elif TORRENT_USE_LOCALE_FILENAMES
			fs::path f = convert_to_native((p / i->path).string());
#else
			fs::path f = p / i->path;
#endif
			// TODO: optimize
			if (exists(f))
#ifndef BOOST_NO_EXCEPTIONS
			try
#endif
			{
				size = file_size(f);
				time = last_write_time(f);
			}
#ifndef BOOST_NO_EXCEPTIONS
			catch (std::exception&) {}
#endif
			sizes.push_back(std::make_pair(size, time));
		}
		return sizes;
	}

	// matches the sizes and timestamps of the files passed in
	// in non-compact mode, actual file sizes and timestamps
	// are allowed to be bigger and more recent than the fast
	// resume data. This is because full allocation will not move
	// pieces, so any older version of the resume data will
	// still be a correct subset of the actual data on disk.
	bool match_filesizes(
		file_storage const& fs
		, fs::path p
		, std::vector<std::pair<size_type, std::time_t> > const& sizes
		, bool compact_mode
		, std::string* error)
	{
		if ((int)sizes.size() != fs.num_files())
		{
			if (error) *error = "mismatching number of files";
			return false;
		}
		p = complete(p);

		std::vector<std::pair<size_type, std::time_t> >::const_iterator s
			= sizes.begin();
		for (file_storage::iterator i = fs.begin()
			, end(fs.end());i != end; ++i, ++s)
		{
			size_type size = 0;
			std::time_t time = 0;
			if (i->pad_file) continue;

#if TORRENT_USE_WPATH
			fs::wpath f = convert_to_wstring((p / i->path).string());
#elif TORRENT_USE_LOCALE_FILENAMES
			fs::path f = convert_to_native((p / i->path).string());
#else
			fs::path f = p / i->path;
#endif
			// TODO: Optimize this! This will result in 3 stat calls per file!
			if (exists(f))
#ifndef BOOST_NO_EXCEPTIONS
			try
#endif
			{
				size = file_size(f);
				time = last_write_time(f);
			}
#ifndef BOOST_NO_EXCEPTIONS
			catch (std::exception&) {}
#endif
			if ((compact_mode && size != s->first)
				|| (!compact_mode && size < s->first))
			{
				if (error) *error = "filesize mismatch for file '"
					+ i->path.external_file_string()
					+ "', size: " + boost::lexical_cast<std::string>(size)
					+ ", expected to be " + boost::lexical_cast<std::string>(s->first)
					+ " bytes";
				return false;
			}
			// allow one second 'slack', because of FAT volumes
			// in sparse mode, allow the files to be more recent
			// than the resume data, but only by 5 minutes
			if ((compact_mode && (time > s->second + 1 || time < s->second - 1)) ||
				(!compact_mode && (time > s->second + 5 * 60 || time < s->second - 1)))
			{
				if (error) *error = "timestamp mismatch for file '"
					+ i->path.external_file_string()
					+ "', modification date: " + boost::lexical_cast<std::string>(time)
					+ ", expected to have modification date "
					+ boost::lexical_cast<std::string>(s->second);
				return false;
			}
		}
		return true;
	}

	// for backwards compatibility, let the default readv and
	// writev implementations be implemented in terms of the
	// old read and write
	int storage_interface::readv(file::iovec_t const* bufs
		, int slot, int offset, int num_bufs)
	{
		int ret = 0;
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
		{
			int r = write((char const*)i->iov_base, slot, offset, i->iov_len);
			offset += i->iov_len;
			if (r == -1) return -1;
			ret += r;
		}
		return ret;
	}

	int storage_interface::writev(file::iovec_t const* bufs, int slot
		, int offset, int num_bufs)
	{
		int ret = 0;
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
		{
			int r = read((char*)i->iov_base, slot, offset, i->iov_len);
			offset += i->iov_len;
			if (r == -1) return -1;
			ret += r;
		}
		return ret;
	}

	int copy_bufs(file::iovec_t const* bufs, int bytes, file::iovec_t* target)
	{
		int size = 0;
		int ret = 1;
		for (;;)
		{
			*target = *bufs;
			size += bufs->iov_len;
			if (size >= bytes)
			{
				target->iov_len -= size - bytes;
				return ret;
			}
			++bufs;
			++target;
			++ret;
		}
	}

	void advance_bufs(file::iovec_t*& bufs, int bytes)
	{
		int size = 0;
		for (;;)
		{
			size += bufs->iov_len;
			if (size >= bytes)
			{
				((char*&)bufs->iov_base) += bufs->iov_len - (size - bytes);
				bufs->iov_len = size - bytes;
				return;
			}
			++bufs;
		}
	}

	int bufs_size(file::iovec_t const* bufs, int num_bufs)
	{
		int size = 0;
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
			size += i->iov_len;
		return size;
	}
	
	void clear_bufs(file::iovec_t const* bufs, int num_bufs)
	{
		for (file::iovec_t const* i = bufs, *end(bufs + num_bufs); i < end; ++i)
			std::memset(i->iov_base, 0, i->iov_len);
	}

#ifdef TORRENT_DEBUG
	int count_bufs(file::iovec_t const* bufs, int bytes)
	{
		int size = 0;
		int count = 1;
		if (bytes == 0) return 0;
		for (file::iovec_t const* i = bufs;; ++i, ++count)
		{
			size += i->iov_len;
			TORRENT_ASSERT(size <= bytes);
			if (size >= bytes) return count;
		}
	}
#endif

	class storage : public storage_interface, boost::noncopyable
	{
	public:
		storage(file_storage const& fs, fs::path const& path, file_pool& fp)
			: m_files(fs)
			, m_pool(fp)
			, m_page_size(4096)
			, m_allocate_files(false)
		{
			TORRENT_ASSERT(m_files.begin() != m_files.end());
			m_save_path = fs::complete(path);
			TORRENT_ASSERT(m_save_path.is_complete());
#ifdef TORRENT_WINDOWS
			SYSTEM_INFO si;
			GetSystemInfo(&si);
			m_page_size = si.dwPageSize;
#else
			m_page_size = sysconf(_SC_PAGESIZE);
#endif
		}

		bool has_any_file();
		bool rename_file(int index, std::string const& new_filename);
		bool release_files();
		bool delete_files();
		bool initialize(bool allocate_files);
		bool move_storage(fs::path save_path);
		int read(char* buf, int slot, int offset, int size);
		int write(char const* buf, int slot, int offset, int size);
		int sparse_end(int start) const;
		int readv(file::iovec_t const* bufs, int slot, int offset, int num_bufs);
		int writev(file::iovec_t const* buf, int slot, int offset, int num_bufs);
		bool move_slot(int src_slot, int dst_slot);
		bool swap_slots(int slot1, int slot2);
		bool swap_slots3(int slot1, int slot2, int slot3);
		bool verify_resume_data(lazy_entry const& rd, std::string& error);
		bool write_resume_data(entry& rd) const;
		sha1_hash hash_for_slot(int slot, partial_hash& ph, int piece_size);

		// this identifies a read or write operation
		// so that storage::readwrite() knows what to
		// do when it's actually touching the file
		struct fileop
		{
			size_type (file::*regular_op)(size_type file_offset
				, file::iovec_t const* bufs, int num_bufs, error_code& ec);
			size_type (storage::*unaligned_op)(boost::shared_ptr<file> const& f
				, size_type file_offset, file::iovec_t const* bufs, int num_bufs
				, error_code& ec);
			int cache_setting;
			int mode;
		};

		void delete_one_file(std::string const& p);
		int readwritev(file::iovec_t const* bufs, int slot, int offset
			, int num_bufs, fileop const&);

		~storage()
		{ m_pool.release(this); }

		size_type read_unaligned(boost::shared_ptr<file> const& file_handle
			, size_type file_offset, file::iovec_t const* bufs, int num_bufs, error_code& ec);
		size_type write_unaligned(boost::shared_ptr<file> const& file_handle
			, size_type file_offset, file::iovec_t const* bufs, int num_bufs, error_code& ec);

		file_storage const& files() const { return m_mapped_files?*m_mapped_files:m_files; }

		boost::scoped_ptr<file_storage> m_mapped_files;
		file_storage const& m_files;

		std::vector<boost::uint8_t> m_file_priority;
		fs::path m_save_path;
		// the file pool is typically stored in
		// the session, to make all storage
		// instances use the same pool
		file_pool& m_pool;

		int m_page_size;
		bool m_allocate_files;
	};

	sha1_hash storage::hash_for_slot(int slot, partial_hash& ph, int piece_size)
	{
		TORRENT_ASSERT(!error());
		int slot_size = piece_size - ph.offset;
		if (slot_size > 0)
		{
			int block_size = 16 * 1024;
			if (disk_pool()) block_size = disk_pool()->block_size();
			int size = slot_size;
			int num_blocks = (size + block_size - 1) / block_size;
			file::iovec_t* bufs = TORRENT_ALLOCA(file::iovec_t, num_blocks);
			for (int i = 0; i < num_blocks; ++i)
			{
				bufs[i].iov_base = disk_pool()->allocate_buffer("hash temp");
				bufs[i].iov_len = (std::min)(block_size, size);
				size -= bufs[i].iov_len;
			}
			readv(bufs, slot, ph.offset, num_blocks);

			for (int i = 0; i < num_blocks; ++i)
			{
				ph.h.update((char const*)bufs[i].iov_base, bufs[i].iov_len);
				disk_pool()->free_buffer((char*)bufs[i].iov_base);
			}
			if (error()) return sha1_hash(0);
		}
		return ph.h.final();
	}

	bool storage::initialize(bool allocate_files)
	{
		m_allocate_files = allocate_files;
		error_code ec;
		// first, create all missing directories
		fs::path last_path;
		for (file_storage::iterator file_iter = files().begin(),
			end_iter = files().end(); file_iter != end_iter; ++file_iter)
		{
			fs::path dir = (m_save_path / file_iter->path).branch_path();

			if (dir != last_path)
			{
				last_path = dir;

#if TORRENT_USE_WPATH
				fs::wpath wp = convert_to_wstring(last_path.string());
				if (!exists(wp))
					create_directories(wp);
#elif TORRENT_USE_LOCALE_FILENAMES
				fs::path p = convert_to_native(last_path.string());
				if (!exists(p))
					create_directories(p);
#else
				if (!exists(last_path))
					create_directories(last_path);
#endif
			}

			int file_index = file_iter - files().begin();

			// ignore files that have priority 0
			if (int(m_file_priority.size()) > file_index
				&& m_file_priority[file_index] == 0) continue;

			// ignore pad files
			if (file_iter->pad_file) continue;

#ifndef BOOST_NO_EXCEPTIONS
			try {
#endif

#if TORRENT_USE_WPATH
			fs::wpath file_path = convert_to_wstring((m_save_path / file_iter->path).string());
#else
			fs::path file_path = m_save_path / file_iter->path;
#endif
			// if the file is empty, just create it either way.
			// if the file already exists, but is larger than what
			// it's supposed to be, also truncate it
			if (allocate_files
				|| file_iter->size == 0
				|| (exists(file_path) && file_size(file_path) > file_iter->size))
			{
				error_code ec;
				int mode = file::read_write;
				if (m_settings
					&& (settings().disk_io_read_mode == session_settings::disable_os_cache
					|| (settings().disk_io_read_mode == session_settings::disable_os_cache_for_aligned_files
					&& ((file_iter->offset + file_iter->file_base) & (m_page_size-1)) == 0)))
					mode |= file::no_buffer;
				if (!m_allocate_files) mode |= file::sparse;
				boost::shared_ptr<file> f = m_pool.open_file(this
					, m_save_path / file_iter->path, mode, ec);
				if (ec) set_error(m_save_path / file_iter->path, ec);
				else if (f)
				{
					f->set_size(file_iter->size, ec);
					if (ec) set_error(m_save_path / file_iter->path, ec);
				}
			}
#ifndef BOOST_NO_EXCEPTIONS
			}
#if BOOST_VERSION >= 103500
			catch (boost::system::system_error& e)
			{
				set_error(m_save_path / file_iter->path, e.code());
				return true;
			}
#else
			catch (boost::filesystem::filesystem_error& e)
			{
				set_error(m_save_path / file_iter->path
					, error_code(e.system_error(), get_system_category()));
				return true;
			}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
		}
		std::vector<boost::uint8_t>().swap(m_file_priority);
		// close files that were opened in write mode
		m_pool.release(this);
		return false;
	}

	bool storage::has_any_file()
	{
		file_storage::iterator i = files().begin();
		file_storage::iterator end = files().end();

		for (; i != end; ++i)
		{
			bool file_exists = false;
#if TORRENT_USE_LOCALE_FILENAMES
			fs::path f = convert_to_native((m_save_path / i->path).string());
#else
			fs::path f = m_save_path / i->path;
#endif
#ifndef BOOST_NO_EXCEPTIONS
			try
			{
#endif
#if TORRENT_USE_WPATH
				fs::wpath wf = convert_to_wstring(f.string());
				file_exists = exists(wf);
#else
				file_exists = exists(f);
#endif
#ifndef BOOST_NO_EXCEPTIONS
			}
#if BOOST_VERSION >= 103500
			catch (boost::system::system_error& e)
			{
				set_error(f, e.code());
				return false;
			}
#else
			catch (boost::filesystem::filesystem_error& e)
			{
				set_error(f, error_code(e.system_error(), get_system_category()));
				return false;
			}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
			if (file_exists && i->size > 0)
				return true;
		}
		return false;
	}

	bool storage::rename_file(int index, std::string const& new_filename)
	{
		if (index < 0 || index >= m_files.num_files()) return true;
		fs::path old_name = m_save_path / files().at(index).path;
		m_pool.release(old_name);

#if TORRENT_USE_WPATH
		fs::wpath old_path = convert_to_wstring(old_name.string());
		fs::wpath new_path = convert_to_wstring((m_save_path / new_filename).string());
#elif TORRENT_USE_LOCALE_FILENAMES
		fs::path const& old_path = convert_to_native(old_name.string());
		fs::path new_path = convert_to_native((m_save_path / new_filename).string());
#else
		fs::path const& old_path = old_name;
		fs::path new_path = m_save_path / new_filename;
#endif

#ifndef BOOST_NO_EXCEPTIONS
		try
		{
#endif
			// if old path doesn't exist, just rename the file
			// in our file_storage, so that when it is created
			// it will get the new name
			create_directories(new_path.branch_path());
			if (exists(old_path)) rename(old_path, new_path);
/*
			error_code ec;
			rename(old_path, new_path, ec);
			if (ec)
			{
				set_error(old_path, ec);
				return;
			}
*/
			if (!m_mapped_files)
			{ m_mapped_files.reset(new file_storage(m_files)); }
			m_mapped_files->rename_file(index, new_filename);
#ifndef BOOST_NO_EXCEPTIONS
		}
#if BOOST_VERSION >= 103500
		catch (boost::system::system_error& e)
		{
			set_error(old_name, e.code());
			return true;
		}
#else
		catch (boost::filesystem::filesystem_error& e)
		{
			set_error(old_name, error_code(e.system_error()
				, get_system_category()));
			return true;
		}
#endif // BOOST_VERSION
#endif
		return false;
	}

	bool storage::release_files()
	{
		m_pool.release(this);
		return false;
	}

	void storage::delete_one_file(std::string const& p)
	{
#if TORRENT_USE_WPATH
#ifndef BOOST_NO_EXCEPTIONS
		try
#endif
		{ fs::remove(convert_to_wstring(p)); }
#ifndef BOOST_NO_EXCEPTIONS
#if BOOST_VERSION >= 103500
		catch (boost::system::system_error& e)
		{
			set_error(p, e.code());
		}
#else
		catch (boost::filesystem::filesystem_error& e)
		{
			set_error(p, error_code(e.system_error(), get_system_category()));
		}
#endif // BOOST_VERSION
#endif // BOOST_NO_EXCEPTIONS
#elif TORRENT_USE_LOCALE_FILENAMES
		if (std::remove(convert_to_native(p).c_str()) != 0 && errno != ENOENT)
{'fraction_non_alphanumeric': 0.13428849561939946, 'fraction_numerical': 0.005391508374310461, 'mean_word_length': 4.0528439904149325, 'pattern_counts': {'":': 0, '<': 131, '<?xml version=': 0, '>': 260, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5238225', 'n_tokens_mistral': 16517, 'n_tokens_neox': 14498, 'n_words': 3611}