<commit_before>//  Copyright (C) 2013, The Duplicati Team

//  http://www.duplicati.com, <EMAIL>
//
//  This library is free software; you can redistribute it and/or modify
//  it under the terms of the GNU Lesser General Public License as
//  published by the Free Software Foundation; either version 2.1 of the
//  License, or (at your option) any later version.
//
//  This library is distributed in the hope that it will be useful, but
//  WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
//  Lesser General Public License for more details.
//
//  You should have received a copy of the GNU Lesser General Public
//  License along with this library; if not, write to the Free Software
//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using System.Linq;
using System.Collections.Generic;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Main.Volumes;
using System.Text;

namespace Duplicati.Library.Main.Operation
{
	internal class CompactHandler
	{
        protected string m_backendurl;
        protected Options m_options;
        protected CompactResults m_result;
        
		public CompactHandler(string backend, Options options, CompactResults result)
		{
            m_backendurl = backend;
            m_options = options;
            m_result = result;
		}
		
		public virtual void Run()
        {
            if (!System.IO.File.Exists(m_options.Dbpath))
                throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
			
            using(var db = new LocalDeleteDatabase(m_options.Dbpath, true))
            {
                var tr = db.BeginTransaction();
                try
                {
                    m_result.SetDatabase(db);
                    Utility.UpdateOptionsFromDb(db, m_options);
                    Utility.VerifyParameters(db, m_options);
    	        	
                    var changed = DoCompact(db, false, ref tr);
                    
                    if (changed && m_options.UploadVerificationFile)
                        FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null);
                    
                    if (!m_options.Dryrun)
                    {
                        using(new Logging.Timer("CommitCompact"))
                            tr.Commit();
                        if (changed)
                        {
                            db.WriteResults();                    
                            db.Vacuum();
                        }
                    }
    				else
    					tr.Rollback();
                    
                    tr = null;
                }
                finally
                {
                    if (tr != null)
                        try { tr.Rollback(); }
                        catch { }
                }
            }
		}
		
		internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction)
        {
            var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
            report.ReportCompactData(m_result);
			
            if (report.ShouldReclaim || report.ShouldCompact)
            {
                using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
                {
                    if (!hasVerifiedBackend && !m_options.NoBackendverification)
                        FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
		
                    BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
                    newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
	
                    IndexVolumeWriter newvolindex = null;
                    if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
                    {
                        newvolindex = new IndexVolumeWriter(m_options);
                        newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
                        db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
                        newvolindex.StartVolume(newvol.RemoteFilename);
                    }
					
                    long blocksInVolume = 0;
                    long discardedBlocks = 0;
                    long discardedSize = 0;
                    byte[] buffer = new byte[m_options.Blocksize];
                    var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
					
                    //These are for bookkeeping
                    var uploadedVolumes = new List<KeyValuePair<string, long>>();
                    var deletedVolumes = new List<KeyValuePair<string, long>>();
                    var downloadedVolumes = new List<KeyValuePair<string, long>>();
					
                    //We start by deleting unused volumes to save space before uploading new stuff
                    var fullyDeleteable = (from v in remoteList
                                           where report.DeleteableVolumes.Contains(v.Name)
                                           select (IRemoteVolume)v).ToList();                    
                    deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));

                    // This list is used to pick up unused volumes,
                    // so they can be deleted once the upload of the
                    // required fragments is complete
                    var deleteableVolumes = new List<IRemoteVolume>();

                    if (report.ShouldCompact)
                    {
                        var volumesToDownload = (from v in remoteList
                                                 where report.CompactableVolumes.Contains(v.Name)
                                                 select (IRemoteVolume)v).ToList();
						
                        using(var q = db.CreateBlockQueryHelper(m_options, transaction))
                        {
                            foreach(var entry in new AsyncDownloader(volumesToDownload, backend))
                            using(var tmpfile = entry.TempFile)
                            {
                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                {
                                    backend.WaitForComplete(db, transaction);
                                    return false;
                                }
                                    
								downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
								var inst = VolumeBase.ParseFilename(entry.Name);
								using(var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
								{
									foreach(var e in f.Blocks)
									{
                                        if (q.UseBlock(e.Key, e.Value, transaction))
										{
											//TODO: How do we get the compression hint? Reverse query for filename in db?
											var s = f.ReadBlock(e.Key, buffer);
											if (s != e.Value)
												throw new Exception("Size mismatch problem, {0} vs {1}");
												
											newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
											if (newvolindex != null)
												newvolindex.AddBlock(e.Key, e.Value);
												
											db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
											blocksInVolume++;
											<commit_msg>Fixed an error message that did not report any information<commit_after>//  Copyright (C) 2013, The Duplicati Team

//  http://www.duplicati.com, <EMAIL>
//
//  This library is free software; you can redistribute it and/or modify
//  it under the terms of the GNU Lesser General Public License as
//  published by the Free Software Foundation; either version 2.1 of the
//  License, or (at your option) any later version.
//
//  This library is distributed in the hope that it will be useful, but
//  WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
//  Lesser General Public License for more details.
//
//  You should have received a copy of the GNU Lesser General Public
//  License along with this library; if not, write to the Free Software
//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using System.Linq;
using System.Collections.Generic;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Main.Volumes;
using System.Text;

namespace Duplicati.Library.Main.Operation
{
	internal class CompactHandler
	{
        protected string m_backendurl;
        protected Options m_options;
        protected CompactResults m_result;
        
		public CompactHandler(string backend, Options options, CompactResults result)
		{
            m_backendurl = backend;
            m_options = options;
            m_result = result;
		}
		
		public virtual void Run()
        {
            if (!System.IO.File.Exists(m_options.Dbpath))
                throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
			
            using(var db = new LocalDeleteDatabase(m_options.Dbpath, true))
            {
                var tr = db.BeginTransaction();
                try
                {
                    m_result.SetDatabase(db);
                    Utility.UpdateOptionsFromDb(db, m_options);
                    Utility.VerifyParameters(db, m_options);
    	        	
                    var changed = DoCompact(db, false, ref tr);
                    
                    if (changed && m_options.UploadVerificationFile)
                        FilelistProcessor.UploadVerificationFile(m_backendurl, m_options, m_result.BackendWriter, db, null);
                    
                    if (!m_options.Dryrun)
                    {
                        using(new Logging.Timer("CommitCompact"))
                            tr.Commit();
                        if (changed)
                        {
                            db.WriteResults();                    
                            db.Vacuum();
                        }
                    }
    				else
    					tr.Rollback();
                    
                    tr = null;
                }
                finally
                {
                    if (tr != null)
                        try { tr.Rollback(); }
                        catch { }
                }
            }
		}
		
		internal bool DoCompact(LocalDeleteDatabase db, bool hasVerifiedBackend, ref System.Data.IDbTransaction transaction)
        {
            var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, transaction);
            report.ReportCompactData(m_result);
			
            if (report.ShouldReclaim || report.ShouldCompact)
            {
                using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db))
                {
                    if (!hasVerifiedBackend && !m_options.NoBackendverification)
                        FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
		
                    BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
                    newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
	
                    IndexVolumeWriter newvolindex = null;
                    if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
                    {
                        newvolindex = new IndexVolumeWriter(m_options);
                        newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
                        db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction);
                        newvolindex.StartVolume(newvol.RemoteFilename);
                    }
					
                    long blocksInVolume = 0;
                    long discardedBlocks = 0;
                    long discardedSize = 0;
                    byte[] buffer = new byte[m_options.Blocksize];
                    var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
					
                    //These are for bookkeeping
                    var uploadedVolumes = new List<KeyValuePair<string, long>>();
                    var deletedVolumes = new List<KeyValuePair<string, long>>();
                    var downloadedVolumes = new List<KeyValuePair<string, long>>();
					
                    //We start by deleting unused volumes to save space before uploading new stuff
                    var fullyDeleteable = (from v in remoteList
                                           where report.DeleteableVolumes.Contains(v.Name)
                                           select (IRemoteVolume)v).ToList();                    
                    deletedVolumes.AddRange(DoDelete(db, backend, fullyDeleteable, ref transaction));

                    // This list is used to pick up unused volumes,
                    // so they can be deleted once the upload of the
                    // required fragments is complete
                    var deleteableVolumes = new List<IRemoteVolume>();

                    if (report.ShouldCompact)
                    {
                        var volumesToDownload = (from v in remoteList
                                                 where report.CompactableVolumes.Contains(v.Name)
                                                 select (IRemoteVolume)v).ToList();
						
                        using(var q = db.CreateBlockQueryHelper(m_options, transaction))
                        {
                            foreach(var entry in new AsyncDownloader(volumesToDownload, backend))
                            using(var tmpfile = entry.TempFile)
                            {
                                if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
                                {
                                    backend.WaitForComplete(db, transaction);
                                    return false;
                                }
                                    
								downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
								var inst = VolumeBase.ParseFilename(entry.Name);
								using(var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
								{
									foreach(var e in f.Blocks)
									{
                                        if (q.UseBlock(e.Key, e.Value, transaction))
										{
											//TODO: How do we get the compression hint? Reverse query for filename in db?
											var s = f.ReadBlock(e.Key, buffer);
											if (s != e.Value)
												throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
												
											newvol.AddBlock(e.Key, buffer, 0, s, Duplicati.Library.Interface.CompressionHint.Compressible);
											if (newvolindex != null)
												newvolindex.AddBlock(e.Key, e.Value);
												
											db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
											blocksInVolume++;
											
{'fraction_non_alphanumeric': 0.07648062701472726, 'fraction_numerical': 0.0034763921370330572, 'mean_word_length': 1.5054631828978622, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7052930', 'n_tokens_mistral': 4271, 'n_tokens_neox': 3671, 'n_words': 973}