<commit_before>/*************************************************************************
 *   Copyright (C) 2011-2013 by <NAME>                       *
 *   paul-louis (at) ageneau (dot) org                                   *
 *                                                                       *
 *   This file is part of Teapotnet.                                     *
 *                                                                       *
 *   Teapotnet is free software: you can redistribute it and/or modify   *
 *   it under the terms of the GNU Affero General Public License as      *
 *   published by the Free Software Foundation, either version 3 of      *
 *   the License, or (at your option) any later version.                 *
 *                                                                       *
 *   Teapotnet is distributed in the hope that it will be useful, but    *
 *   WITHOUT ANY WARRANTY; without even the implied warranty of          *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the        *
 *   GNU Affero General Public License for more details.                 *
 *                                                                       *
 *   You should have received a copy of the GNU Affero General Public    *
 *   License along with Teapotnet.                                       *
 *   If not, see <http://www.gnu.org/licenses/>.                         *
 *************************************************************************/

#include "tpn/resource.h"
#include "tpn/store.h"
#include "tpn/config.h"

#include "pla/binaryserializer.h"

namespace tpn
{

Resource::Resource(void) :
	mIndexBlock(NULL),
	mIndexRecord(NULL)
{

}

Resource::Resource(const Resource &resource) :
	mIndexBlock(NULL),
	mIndexRecord(NULL)
{
	*this = resource;
}

Resource::Resource(const BinaryString &digest, bool localOnly) :
	mIndexBlock(NULL),
	mIndexRecord(NULL)
{
	fetch(digest, localOnly);
}

Resource::~Resource(void)
{
	delete mIndexBlock;
	delete mIndexRecord;
}

void Resource::fetch(const BinaryString &digest, bool localOnly)
{
	delete mIndexBlock;
	delete mIndexRecord;
	mIndexRecord = NULL;
	mIndexBlock = NULL;
	
	if(localOnly && !Store::Instance->hasBlock(digest))
		throw Exception(String("Local resource not found: ") + digest.toString());
	
	//LogDebug("Resource::fetch", "Fetching resource " + digest.toString());
	
	try {
		mIndexBlock = new Block(digest);
		mIndexRecord = new IndexRecord;
		
		//LogDebug("Resource::fetch", "Reading index block for " + digest.toString());
		
		BinarySerializer serializer(mIndexBlock);
		AssertIO(static_cast<Serializer*>(&serializer)->input(mIndexRecord));
	}
	catch(const std::exception &e)
	{
		delete mIndexBlock;
		delete mIndexRecord;
		mIndexRecord = NULL;
		mIndexBlock = NULL;
		throw Exception(String("Unable to fetch resource index block: ") + e.what());
	}

}

void Resource::process(const String &filename, const String &name, const String &type, const String &secret)
{
	BinaryString salt;
	
	// If secret not empty then this is an encrypted resource
	if(!secret.empty())
	{
		Random rnd;
		rnd.read(salt, 32);	// 256 bits
	}
	
	// Fill index record
	delete mIndexRecord;
	int64_t size = File::Size(filename);
	mIndexRecord = new Resource::IndexRecord;
	mIndexRecord->name = name;
	mIndexRecord->type = type;
	mIndexRecord->size = size;
	mIndexRecord->salt = salt;
	mIndexRecord->blockDigests.reserve(size/Block::Size);
	
	// Process blocks
	File file(filename, File::Read);
	BinaryString blockDigest;
	
	if(!secret.empty())
	{
		BinaryString key;
		Sha256().pbkdf2_hmac(secret, salt, key, 32, 100000);
		
		VAR(name);
		VAR(type);
		VAR(key);
		VAR(salt);
		
		uint64_t i = 0;
		while(true)
		{
			BinaryString subsalt;
			subsalt.writeBinary(i);
			
			// Generate subkey
			BinaryString subkey;
			Sha256().pbkdf2_hmac(key, subsalt, subkey, 32, 100);
			
			// Generate iv
			BinaryString iv;
			Sha256().pbkdf2_hmac(salt, subsalt, iv, 16, 100);
			
			VAR(i);
			VAR(subkey);<commit_msg>Changed resource encryption to deterministic<commit_after>/*************************************************************************
 *   Copyright (C) 2011-2013 by <NAME>                       *
 *   paul-louis (at) ageneau (dot) org                                   *
 *                                                                       *
 *   This file is part of Teapotnet.                                     *
 *                                                                       *
 *   Teapotnet is free software: you can redistribute it and/or modify   *
 *   it under the terms of the GNU Affero General Public License as      *
 *   published by the Free Software Foundation, either version 3 of      *
 *   the License, or (at your option) any later version.                 *
 *                                                                       *
 *   Teapotnet is distributed in the hope that it will be useful, but    *
 *   WITHOUT ANY WARRANTY; without even the implied warranty of          *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the        *
 *   GNU Affero General Public License for more details.                 *
 *                                                                       *
 *   You should have received a copy of the GNU Affero General Public    *
 *   License along with Teapotnet.                                       *
 *   If not, see <http://www.gnu.org/licenses/>.                         *
 *************************************************************************/

#include "tpn/resource.h"
#include "tpn/store.h"
#include "tpn/config.h"

#include "pla/binaryserializer.h"

namespace tpn
{

Resource::Resource(void) :
	mIndexBlock(NULL),
	mIndexRecord(NULL)
{

}

Resource::Resource(const Resource &resource) :
	mIndexBlock(NULL),
	mIndexRecord(NULL)
{
	*this = resource;
}

Resource::Resource(const BinaryString &digest, bool localOnly) :
	mIndexBlock(NULL),
	mIndexRecord(NULL)
{
	fetch(digest, localOnly);
}

Resource::~Resource(void)
{
	delete mIndexBlock;
	delete mIndexRecord;
}

void Resource::fetch(const BinaryString &digest, bool localOnly)
{
	delete mIndexBlock;
	delete mIndexRecord;
	mIndexRecord = NULL;
	mIndexBlock = NULL;
	
	if(localOnly && !Store::Instance->hasBlock(digest))
		throw Exception(String("Local resource not found: ") + digest.toString());
	
	//LogDebug("Resource::fetch", "Fetching resource " + digest.toString());
	
	try {
		mIndexBlock = new Block(digest);
		mIndexRecord = new IndexRecord;
		
		//LogDebug("Resource::fetch", "Reading index block for " + digest.toString());
		
		BinarySerializer serializer(mIndexBlock);
		AssertIO(static_cast<Serializer*>(&serializer)->input(mIndexRecord));
	}
	catch(const std::exception &e)
	{
		delete mIndexBlock;
		delete mIndexRecord;
		mIndexRecord = NULL;
		mIndexBlock = NULL;
		throw Exception(String("Unable to fetch resource index block: ") + e.what());
	}

}

void Resource::process(const String &filename, const String &name, const String &type, const String &secret)
{
	BinaryString salt;
	
	// If secret is not empty then this is an encrypted resource
	if(!secret.empty())
	{
		// Generate salt from plaintext
		// Because we need to be able to recognize data is identical even when encrypted
		BinaryString digest;
		File file(filename, File::Read);
		Sha256().compute(file, digest);
		Sha256().pbkdf2_hmac(salt, type, digest, 32, 100000);
	}
	
	// Fill index record
	delete mIndexRecord;
	int64_t size = File::Size(filename);
	mIndexRecord = new Resource::IndexRecord;
	mIndexRecord->name = name;
	mIndexRecord->type = type;
	mIndexRecord->size = size;
	mIndexRecord->salt = salt;
	mIndexRecord->blockDigests.reserve(size/Block::Size);
	
	// Process blocks
	File file(filename, File::Read);
	BinaryString blockDigest;
	
	if(!secret.empty())
	{
		BinaryString key;
		Sha256().pbkdf2_hmac(secret, salt, key, 32, 100000);
		
		uint64_t i = 0;
		while(true)
		{
			BinaryString subsalt;
			subsalt.writeBinary(i);
			
			// Generate subkey
			BinaryString subkey;
			Sha256().pbkdf2_hmac(key, subsalt, subkey, 32, 100);
			
			// Generate iv
			BinaryString iv;
			Sha256().pbkdf2_hmac(salt, subsalt, iv, 16, 100);
			
			VAR(i);
			VAR(subkey);
{'fraction_non_alphanumeric': 0.1383250273489729, 'fraction_numerical': 0.013127506989181963, 'mean_word_length': 2.4820143884892087, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3257738', 'n_tokens_mistral': 2572, 'n_tokens_neox': 2284, 'n_words': 642}