<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Duplicati.Library.Main.Volumes;

namespace Duplicati.Library.Main.Database
{
    public partial class LocalRestoreDatabase : LocalDatabase
    {
        protected string m_tempfiletable;
        protected string m_tempblocktable;
        protected long m_blocksize;

        public LocalRestoreDatabase(string path, long blocksize)
            : this(new LocalDatabase(path, "Restore"), blocksize)
        {
        }

        public LocalRestoreDatabase(LocalDatabase dbparent, long blocksize)
            : base(dbparent)
        {
            //TODO: Should read this from DB?
            m_blocksize = blocksize;
        }

        public void PrepareRestoreFilelist(DateTime restoretime, Library.Utility.IFilter filter, CommunicationStatistics stat)
		{
			var guid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());

			m_tempfiletable = "Fileset-" + guid;
			m_tempblocktable = "Blocks-" + guid;

            using (var cmd = m_connection.CreateCommand())
            {
                long filesetId = GetFilesetID(restoretime);

                cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable);
                cmd.ExecuteNonQuery();

                cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL)", m_tempblocktable);
                cmd.ExecuteNonQuery();

                if (filter == null || filter.Empty)
                {
                    // Simple case, restore everything
                    cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable);
                    cmd.AddParameter(filesetId);
                    cmd.ExecuteNonQuery();
                }
                else if (filter is Library.Utility.FilterExpression && (filter as Library.Utility.FilterExpression).Type == Duplicati.Library.Utility.FilterType.Simple)
                {
	                // If we get a list of filenames, the lookup table is faster
                    using(var tr = m_connection.BeginTransaction())
                    {
                    	var p = (filter as Library.Utility.FilterExpression).GetSimpleList();
                        var m_filenamestable = "Filenames-" + guid;
                        cmd.Transaction = tr;
                        cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable));
                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable);
                        cmd.AddParameter();
                        
                        foreach (var s in p)
                        {
                            cmd.SetParameterValue(0, s);
                            cmd.ExecuteNonQuery();
                        }
                        
                        //TODO: Handle case-insensitive filename lookup
                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable);
                        cmd.SetParameterValue(0, filesetId);
                        var c = cmd.ExecuteNonQuery();
                        
                        cmd.Parameters.Clear();
                        
                        if (c != p.Length)
                        {
                            var sb = new StringBuilder();
                            sb.AppendLine();
                            
                            using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable)))
                                while(rd.Read())
                                    sb.AppendLine(rd.GetValue(0).ToString());

                            DateTime actualrestoretime = Convert.ToDateTime(cmd.ExecuteScalar(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", filesetId));
                            stat.LogWarning(string.Format("{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb), null);
                            cmd.Parameters.Clear();
                        }
                        
                        cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_filenamestable));
                        
                        tr.Commit();
                    }
                }
                else
                {
                    // Restore but filter elements based on the filter expression
                    // If this is too slow, we could add a special handler for wildcard searches too
                    cmd.CommandText = string.Format(@"SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = ?");
                    cmd.AddParameter(filesetId);

                    object[] values = new object[3];
                    using (var cmd2 = m_connection.CreateCommand())
                    {
                        cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?)", m_tempfiletable);
                        cmd2.AddParameter();
                        cmd2.AddParameter();
                        cmd2.AddParameter();

                        using (var rd = cmd.ExecuteReader())
                            while (rd.Read())
                            {
                                rd.GetValues(values);
                                if (values[0] != null && filter.Matches(values[0].ToString()))
                                {
                                    cmd2.SetParameterValue(0, values[0]);
                                    cmd2.SetParameterValue(1, values[0]);
                                    cmd2.SetParameterValue(2, values[0]);
                                    cmd2.ExecuteNonQuery();
                                }
                            }
                    }
                }
            }
        }

        public string GetLargestPrefix()
        {
            using (var cmd = m_connection.CreateCommand())
            {
                cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable);
                var v0 = cmd.ExecuteScalar();<commit_msg>Bugfix for restore with regexp<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Duplicati.Library.Main.Volumes;

namespace Duplicati.Library.Main.Database
{
    public partial class LocalRestoreDatabase : LocalDatabase
    {
        protected string m_tempfiletable;
        protected string m_tempblocktable;
        protected long m_blocksize;

        public LocalRestoreDatabase(string path, long blocksize)
            : this(new LocalDatabase(path, "Restore"), blocksize)
        {
        }

        public LocalRestoreDatabase(LocalDatabase dbparent, long blocksize)
            : base(dbparent)
        {
            //TODO: Should read this from DB?
            m_blocksize = blocksize;
        }

        public void PrepareRestoreFilelist(DateTime restoretime, Library.Utility.IFilter filter, CommunicationStatistics stat)
		{
			var guid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());

			m_tempfiletable = "Fileset-" + guid;
			m_tempblocktable = "Blocks-" + guid;

            using (var cmd = m_connection.CreateCommand())
            {
                long filesetId = GetFilesetID(restoretime);

                cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable);
                cmd.ExecuteNonQuery();

                cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL)", m_tempblocktable);
                cmd.ExecuteNonQuery();

                if (filter == null || filter.Empty)
                {
                    // Simple case, restore everything
                    cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? ", m_tempfiletable);
                    cmd.AddParameter(filesetId);
                    cmd.ExecuteNonQuery();
                }
                else if (filter is Library.Utility.FilterExpression && (filter as Library.Utility.FilterExpression).Type == Duplicati.Library.Utility.FilterType.Simple)
                {
	                // If we get a list of filenames, the lookup table is faster
                    using(var tr = m_connection.BeginTransaction())
                    {
                    	var p = (filter as Library.Utility.FilterExpression).GetSimpleList();
                        var m_filenamestable = "Filenames-" + guid;
                        cmd.Transaction = tr;
                        cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", m_filenamestable));
                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", m_filenamestable);
                        cmd.AddParameter();
                        
                        foreach (var s in p)
                        {
                            cmd.SetParameterValue(0, s);
                            cmd.ExecuteNonQuery();
                        }
                        
                        //TODO: Handle case-insensitive filename lookup
                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable);
                        cmd.SetParameterValue(0, filesetId);
                        var c = cmd.ExecuteNonQuery();
                        
                        cmd.Parameters.Clear();
                        
                        if (c != p.Length)
                        {
                            var sb = new StringBuilder();
                            sb.AppendLine();
                            
                            using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_filenamestable, m_tempfiletable)))
                                while(rd.Read())
                                    sb.AppendLine(rd.GetValue(0).ToString());

                            DateTime actualrestoretime = Convert.ToDateTime(cmd.ExecuteScalar(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", filesetId));
                            stat.LogWarning(string.Format("{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb), null);
                            cmd.Parameters.Clear();
                        }
                        
                        cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_filenamestable));
                        
                        tr.Commit();
                    }
                }
                else
                {
                    // Restore but filter elements based on the filter expression
                    // If this is too slow, we could add a special handler for wildcard searches too
                    cmd.CommandText = string.Format(@"SELECT ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = ?");
                    cmd.AddParameter(filesetId);

                    object[] values = new object[3];
                    using (var cmd2 = m_connection.CreateCommand())
                    {
                        cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?)", m_tempfiletable);
                        cmd2.AddParameter();
                        cmd2.AddParameter();
                        cmd2.AddParameter();

                        using (var rd = cmd.ExecuteReader())
                            while (rd.Read())
                            {
                                rd.GetValues(values);
                                if (values[0] != null && filter.Matches(values[0].ToString()))
                                {
                                    cmd2.SetParameterValue(0, values[0]);
                                    cmd2.SetParameterValue(1, values[1]);
                                    cmd2.SetParameterValue(2, values[2]);
                                    cmd2.ExecuteNonQuery();
                                }
                            }
                    }
                }
            }
        }

        public string GetLargestPrefix()
        {
            using (var cmd = m_connection.CreateCommand())
            {
                cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable);
                var v0 = cmd.ExecuteScalar();
{'fraction_non_alphanumeric': 0.13058512441521458, 'fraction_numerical': 0.005152891721472643, 'mean_word_length': 1.57282400139543, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4260266', 'n_tokens_mistral': 3856, 'n_tokens_neox': 3558, 'n_words': 849}