<commit_before>#include "resources/asset_pack.h"
#include "resources/asset_database.h"
#include "halley/resources/resource_data.h"
#include "halley/bytes/byte_serializer.h"
#include "halley/bytes/compression.h"
#include "halley/maths/random.h"
#include "halley/utils/encrypt.h"

using namespace Halley;

void AssetPackHeader::init(size_t assetDbSize)
{
	memcpy(identifier.data(), "HALLEYPK", 8);
	assetDbStartPos = sizeof(AssetPackHeader);
	dataStartPos = assetDbStartPos + assetDbSize;
	memset(iv.data(), 0, iv.size());
}

AssetPack::AssetPack()
	: assetDb(std::make_unique<AssetDatabase>())
	, hasReader(false)
{
	memset(iv.data(), 0, iv.size());
}

AssetPack::AssetPack(AssetPack&& other)
{
	*this = std::move(other);
}

AssetPack::AssetPack(std::unique_ptr<ResourceDataReader> _reader, const String& encryptionKey, bool preLoad)
	: reader(std::move(_reader))
	, hasReader(true)
{
	// Read header
	size_t totalSize = reader->size();
	if (totalSize < sizeof(AssetPackHeader)) {
		throw Exception("Asset pack is invalid (too small)", HalleyExceptions::Resources);
	}
	AssetPackHeader header;
	int nRead = reader->read(gsl::as_writeable_bytes(gsl::span<AssetPackHeader>(&header, 1)));
	if (nRead != int(sizeof(header))) {
		throw Exception("Unable to read header", HalleyExceptions::Resources);
	}
	if (memcmp(header.identifier.data(), "HALLEYPK", 8) != 0) {
		throw Exception("Asset pack is invalid (invalid identifier)", HalleyExceptions::Resources);
	}
	iv = header.iv;
	dataOffset = size_t(header.dataStartPos);

	// Read asset database
	{
		const size_t assetDbSize = size_t(header.dataStartPos - header.assetDbStartPos);
		auto assetDbBytes = Bytes(assetDbSize);
		nRead = reader->read(gsl::as_writeable_bytes(gsl::span<Byte>(assetDbBytes)));
		if (nRead != int(assetDbBytes.size())) {
			throw Exception("Unable to read header", HalleyExceptions::Resources);
		}
		assetDb = std::make_unique<AssetDatabase>();
		Deserializer::fromBytes<AssetDatabase>(*assetDb, Compression::decompress(assetDbBytes));
	}

	if (preLoad || !encryptionKey.isEmpty()) {
		readToMemory();
	}

	if (!encryptionKey.isEmpty()) {
		decrypt(encryptionKey);
	}
}

AssetPack::~AssetPack()
{
}

AssetPack& AssetPack::operator=(AssetPack&& other)
{
	std::unique_lock<std::mutex> lock(other.readerMutex);

	assetDb = std::move(other.assetDb);
	dataOffset = other.dataOffset;
	reader = std::move(other.reader);
	data = std::move(other.data);
	hasReader = !!reader;

	other.hasReader = false;
	other.reader.reset();

	return *this;
}

AssetDatabase& AssetPack::getAssetDatabase()
{
	return *assetDb;
}

const AssetDatabase& AssetPack::getAssetDatabase() const<commit_msg>Optional encryption for assets.<commit_after>#include "resources/asset_pack.h"
#include "resources/asset_database.h"
#include "halley/resources/resource_data.h"
#include "halley/bytes/byte_serializer.h"
#include "halley/bytes/compression.h"
#include "halley/maths/random.h"
#include "halley/utils/encrypt.h"

using namespace Halley;

void AssetPackHeader::init(size_t assetDbSize)
{
	memcpy(identifier.data(), "HALLEYPK", 8);
	assetDbStartPos = sizeof(AssetPackHeader);
	dataStartPos = assetDbStartPos + assetDbSize;
	memset(iv.data(), 0, iv.size());
}

AssetPack::AssetPack()
	: assetDb(std::make_unique<AssetDatabase>())
	, hasReader(false)
{
	memset(iv.data(), 0, iv.size());
}

AssetPack::AssetPack(AssetPack&& other)
{
	*this = std::move(other);
}

AssetPack::AssetPack(std::unique_ptr<ResourceDataReader> _reader, const String& encryptionKey, bool preLoad)
	: reader(std::move(_reader))
	, hasReader(true)
{
	// Read header
	size_t totalSize = reader->size();
	if (totalSize < sizeof(AssetPackHeader)) {
		throw Exception("Asset pack is invalid (too small)", HalleyExceptions::Resources);
	}
	AssetPackHeader header;
	int nRead = reader->read(gsl::as_writeable_bytes(gsl::span<AssetPackHeader>(&header, 1)));
	if (nRead != int(sizeof(header))) {
		throw Exception("Unable to read header", HalleyExceptions::Resources);
	}
	if (memcmp(header.identifier.data(), "HALLEYPK", 8) != 0) {
		throw Exception("Asset pack is invalid (invalid identifier)", HalleyExceptions::Resources);
	}
	iv = header.iv;
	dataOffset = size_t(header.dataStartPos);

	// Read asset database
	{
		const size_t assetDbSize = size_t(header.dataStartPos - header.assetDbStartPos);
		auto assetDbBytes = Bytes(assetDbSize);
		nRead = reader->read(gsl::as_writeable_bytes(gsl::span<Byte>(assetDbBytes)));
		if (nRead != int(assetDbBytes.size())) {
			throw Exception("Unable to read header", HalleyExceptions::Resources);
		}
		assetDb = std::make_unique<AssetDatabase>();
		Deserializer::fromBytes<AssetDatabase>(*assetDb, Compression::decompress(assetDbBytes));
	}

	std::array<char, 16> ivEmpty;
	memset(ivEmpty.data(), 0, ivEmpty.size());
	const bool hasCrypt = memcmp(iv.data(), ivEmpty.data(), iv.size()) != 0 && !encryptionKey.isEmpty();

	if (preLoad || hasCrypt) {
		readToMemory();
	}

	if (hasCrypt) {
		decrypt(encryptionKey);
	}
}

AssetPack::~AssetPack()
{
}

AssetPack& AssetPack::operator=(AssetPack&& other)
{
	std::unique_lock<std::mutex> lock(other.readerMutex);

	assetDb = std::move(other.assetDb);
	dataOffset = other.dataOffset;
	reader = std::move(other.reader);
	data = std::move(other.data);
	hasReader = !!reader;

	other.hasReader = false;
	other.reader.reset();

	return *this;
}

AssetDatabase& AssetPack::getAssetDatabase()
{
	return *assetDb;
}

const AssetDatabase& AssetPack::getAssetDatabase() const
{'fraction_non_alphanumeric': 0.1582905355832722, 'fraction_numerical': 0.00293470286133529, 'mean_word_length': 6.531767955801105, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6072130', 'n_tokens_mistral': 1915, 'n_tokens_neox': 1853, 'n_words': 336}