<commit_before>/*************************************************************************
*    UrBackup - Client/Server backup system
*    Copyright (C) 2011-2016 <NAME>
*
*    This program is free software: you can redistribute it and/or modify
*    it under the terms of the GNU Affero General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU Affero General Public License for more details.
*
*    You should have received a copy of the GNU Affero General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/

#include "ChangeJournalWatcher.h"
#include "../urbackupcommon/os_functions.h"
#include "../Interface/Server.h"
#include "../Interface/DatabaseCursor.h"
#include "../stringtools.h"
#include "DirectoryWatcherThread.h"
#include <memory>
#include <assert.h>


namespace usn
{
	typedef struct {
		DWORD         RecordLength;
		WORD          MajorVersion;
		WORD          MinorVersion;
		BYTE          FileReferenceNumber[16];
		BYTE          ParentFileReferenceNumber[16];
		USN           Usn;
		LARGE_INTEGER TimeStamp;
		DWORD         Reason;
		DWORD         SourceInfo;
		DWORD         SecurityId;
		DWORD         FileAttributes;
		WORD          FileNameLength;
		WORD          FileNameOffset;
		WCHAR         FileName[1];
	} USN_RECORD_V3, *PUSN_RECORD_V3;

	typedef struct {
		DWORDLONG StartFileReferenceNumber;
		USN       LowUsn;
		USN       HighUsn;
		WORD      MinMajorVersion;
		WORD      MaxMajorVersion;
	} MFT_ENUM_DATA_V1, *PMFT_ENUM_DATA_V1;

	UsnInt get_usn_record(PUSN_RECORD usn_record)
	{
		if(usn_record->MajorVersion==2)
		{
			std::wstring filename;
			filename.resize(usn_record->FileNameLength / sizeof(wchar_t) );
			memcpy(&filename[0], (PBYTE) usn_record + usn_record->FileNameOffset, usn_record->FileNameLength);

			UsnInt ret = {
				2,
				uint128(usn_record->FileReferenceNumber),
				uint128(usn_record->ParentFileReferenceNumber),
				usn_record->Usn,
				usn_record->Reason,
				Server->ConvertFromWchar(filename),
				0,
				usn_record->FileAttributes
			};

			return ret;
		}
		else if(usn_record->MajorVersion==3)
		{
			PUSN_RECORD_V3 usn_record_v3 = (PUSN_RECORD_V3)usn_record;
			std::wstring filename;
			filename.resize(usn_record_v3->FileNameLength / sizeof(wchar_t) );
			memcpy(&filename[0], (PBYTE) usn_record_v3 + usn_record_v3->FileNameOffset, usn_record_v3->FileNameLength);

			UsnInt ret = {
				3,
				uint128(usn_record_v3->FileReferenceNumber),
				uint128(usn_record_v3->ParentFileReferenceNumber),
				usn_record_v3->Usn,
				usn_record_v3->Reason,
				Server->ConvertFromWchar(filename),
				0,
				usn_record_v3->FileAttributes
			};

			return ret;
		}
		else
		{
			assert("USN record does not have major version 2 or 3");
		}

		return UsnInt();
	}

	bool enum_usn_data(HANDLE hVolume, DWORDLONG StartFileReferenceNumber, BYTE* pData, DWORD dataSize, DWORD& firstError, DWORD& version, DWORD& cb)
	{
		if(version==0)
		{
			MFT_ENUM_DATA_V0 med;
			med.StartFileReferenceNumber = StartFileReferenceNumber;
			med.LowUsn = 0;
			med.HighUsn = MAXLONGLONG;

			BOOL b = DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, &med, sizeof(med), pData, dataSize, &cb, NULL);

			if(b==FALSE)
			{
				firstError=GetLastError();
				version=1;
				return enum_usn_data(hVolume, StartFileReferenceNumber, pData, dataSize, firstError, version, cb);
			}

			return true;
		}
		else if(version==1)
		{
			usn::MFT_ENUM_DATA_V1 med_v1;
			med_v1.StartFileReferenceNumber = StartFileReferenceNumber;
			med_v1.LowUsn = 0;
			med_v1.HighUsn = MAXLONGLONG;

			BOOL b = DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, &med_v1, sizeof(med_v1), pData, dataSize, &cb, NULL);

			return b!=FALSE;
		}
		else
		{
			Server->Log("Unknown usn data version " + convert((int)version), LL_ERROR);
			assert(false);
			return FALSE;
		}
	}

	uint64 atoiu64(const std::string& str)
	{
		return static_cast<uint64>(watoi64(str));
	}
}

const DWORDLONG usn_reindex_num=1000000; // one million

//#define MFT_ON_DEMAND_LOOKUP

ChangeJournalWatcher::ChangeJournalWatcher(DirectoryWatcherThread * dwt, IDatabase *pDB)
	: dwt(dwt), db(pDB), last_backup_time(0), journal_dao(pDB), unsupported_usn_version_err(false), freeze_open_write_files(false)
{
	indexing_in_progress=false;
	has_error=false;
	last_index_update=0;

	usn_logging_enabled = Server->getServerParameter("usn_logging_enabled")=="true" ||
		FileExists("usn_logging_enabled");
}

ChangeJournalWatcher::~ChangeJournalWatcher(void)
{
	for(std::map<std::string, SChangeJournal>::iterator it=wdirs.begin();it!=wdirs.end();++it)
	{
		CloseHandle(it->second.hVolume);
	}
}

void ChangeJournalWatcher::deleteJournalData(const std::string &vol)
{
	journal_dao.delJournalData(vol);
}

void ChangeJournalWatcher::deleteJournalId(const std::string &vol)
{
	journal_dao.delJournalDeviceId(vol);
}

void ChangeJournalWatcher::applySavedJournalData(const std::string & vol, SChangeJournal &cj, std::map<std::string, bool>& local_open_write_files, bool& started_transaction)
{
	if (!journal_dao.getJournalDataSingle(vol).exists)
	{
		return;
	}

	Server->Log("Applying saved journal data...", LL_DEBUG);
	DBScopedWriteTransaction write_transaction(NULL);
	if (!started_transaction)
	{
		write_transaction.reset(db);
	}
	started_transaction = false;
	
	{<commit_msg>Fix nested transaction starting<commit_after>/*************************************************************************
*    UrBackup - Client/Server backup system
*    Copyright (C) 2011-2016 <NAME>
*
*    This program is free software: you can redistribute it and/or modify
*    it under the terms of the GNU Affero General Public License as published by
*    the Free Software Foundation, either version 3 of the License, or
*    (at your option) any later version.
*
*    This program is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*    GNU Affero General Public License for more details.
*
*    You should have received a copy of the GNU Affero General Public License
*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**************************************************************************/

#include "ChangeJournalWatcher.h"
#include "../urbackupcommon/os_functions.h"
#include "../Interface/Server.h"
#include "../Interface/DatabaseCursor.h"
#include "../stringtools.h"
#include "DirectoryWatcherThread.h"
#include <memory>
#include <assert.h>


namespace usn
{
	typedef struct {
		DWORD         RecordLength;
		WORD          MajorVersion;
		WORD          MinorVersion;
		BYTE          FileReferenceNumber[16];
		BYTE          ParentFileReferenceNumber[16];
		USN           Usn;
		LARGE_INTEGER TimeStamp;
		DWORD         Reason;
		DWORD         SourceInfo;
		DWORD         SecurityId;
		DWORD         FileAttributes;
		WORD          FileNameLength;
		WORD          FileNameOffset;
		WCHAR         FileName[1];
	} USN_RECORD_V3, *PUSN_RECORD_V3;

	typedef struct {
		DWORDLONG StartFileReferenceNumber;
		USN       LowUsn;
		USN       HighUsn;
		WORD      MinMajorVersion;
		WORD      MaxMajorVersion;
	} MFT_ENUM_DATA_V1, *PMFT_ENUM_DATA_V1;

	UsnInt get_usn_record(PUSN_RECORD usn_record)
	{
		if(usn_record->MajorVersion==2)
		{
			std::wstring filename;
			filename.resize(usn_record->FileNameLength / sizeof(wchar_t) );
			memcpy(&filename[0], (PBYTE) usn_record + usn_record->FileNameOffset, usn_record->FileNameLength);

			UsnInt ret = {
				2,
				uint128(usn_record->FileReferenceNumber),
				uint128(usn_record->ParentFileReferenceNumber),
				usn_record->Usn,
				usn_record->Reason,
				Server->ConvertFromWchar(filename),
				0,
				usn_record->FileAttributes
			};

			return ret;
		}
		else if(usn_record->MajorVersion==3)
		{
			PUSN_RECORD_V3 usn_record_v3 = (PUSN_RECORD_V3)usn_record;
			std::wstring filename;
			filename.resize(usn_record_v3->FileNameLength / sizeof(wchar_t) );
			memcpy(&filename[0], (PBYTE) usn_record_v3 + usn_record_v3->FileNameOffset, usn_record_v3->FileNameLength);

			UsnInt ret = {
				3,
				uint128(usn_record_v3->FileReferenceNumber),
				uint128(usn_record_v3->ParentFileReferenceNumber),
				usn_record_v3->Usn,
				usn_record_v3->Reason,
				Server->ConvertFromWchar(filename),
				0,
				usn_record_v3->FileAttributes
			};

			return ret;
		}
		else
		{
			assert("USN record does not have major version 2 or 3");
		}

		return UsnInt();
	}

	bool enum_usn_data(HANDLE hVolume, DWORDLONG StartFileReferenceNumber, BYTE* pData, DWORD dataSize, DWORD& firstError, DWORD& version, DWORD& cb)
	{
		if(version==0)
		{
			MFT_ENUM_DATA_V0 med;
			med.StartFileReferenceNumber = StartFileReferenceNumber;
			med.LowUsn = 0;
			med.HighUsn = MAXLONGLONG;

			BOOL b = DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, &med, sizeof(med), pData, dataSize, &cb, NULL);

			if(b==FALSE)
			{
				firstError=GetLastError();
				version=1;
				return enum_usn_data(hVolume, StartFileReferenceNumber, pData, dataSize, firstError, version, cb);
			}

			return true;
		}
		else if(version==1)
		{
			usn::MFT_ENUM_DATA_V1 med_v1;
			med_v1.StartFileReferenceNumber = StartFileReferenceNumber;
			med_v1.LowUsn = 0;
			med_v1.HighUsn = MAXLONGLONG;

			BOOL b = DeviceIoControl(hVolume, FSCTL_ENUM_USN_DATA, &med_v1, sizeof(med_v1), pData, dataSize, &cb, NULL);

			return b!=FALSE;
		}
		else
		{
			Server->Log("Unknown usn data version " + convert((int)version), LL_ERROR);
			assert(false);
			return FALSE;
		}
	}

	uint64 atoiu64(const std::string& str)
	{
		return static_cast<uint64>(watoi64(str));
	}
}

const DWORDLONG usn_reindex_num=1000000; // one million

//#define MFT_ON_DEMAND_LOOKUP

ChangeJournalWatcher::ChangeJournalWatcher(DirectoryWatcherThread * dwt, IDatabase *pDB)
	: dwt(dwt), db(pDB), last_backup_time(0), journal_dao(pDB), unsupported_usn_version_err(false), freeze_open_write_files(false)
{
	indexing_in_progress=false;
	has_error=false;
	last_index_update=0;

	usn_logging_enabled = Server->getServerParameter("usn_logging_enabled")=="true" ||
		FileExists("usn_logging_enabled");
}

ChangeJournalWatcher::~ChangeJournalWatcher(void)
{
	for(std::map<std::string, SChangeJournal>::iterator it=wdirs.begin();it!=wdirs.end();++it)
	{
		CloseHandle(it->second.hVolume);
	}
}

void ChangeJournalWatcher::deleteJournalData(const std::string &vol)
{
	journal_dao.delJournalData(vol);
}

void ChangeJournalWatcher::deleteJournalId(const std::string &vol)
{
	journal_dao.delJournalDeviceId(vol);
}

void ChangeJournalWatcher::applySavedJournalData(const std::string & vol, SChangeJournal &cj, std::map<std::string, bool>& local_open_write_files, bool& started_transaction)
{
	if (!journal_dao.getJournalDataSingle(vol).exists)
	{
		return;
	}

	Server->Log("Applying saved journal data...", LL_DEBUG);
	DBScopedWriteTransaction write_transaction(NULL);
	if (!started_transaction)
	{
		write_transaction.reset(db);
	}
	
	{
{'fraction_non_alphanumeric': 0.1312952818519173, 'fraction_numerical': 0.01449019261353596, 'mean_word_length': 4.364454976303318, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2220344', 'n_tokens_mistral': 4279, 'n_tokens_neox': 3825, 'n_words': 777}