<commit_before>#region Disclaimer / License
// Copyright (C) 2011, <NAME>
// http://www.hexad.dk, <EMAIL>
// 
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
// 
#endregion
using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.Win32.SafeHandles;
using System.Runtime.InteropServices;
using System.ComponentModel;

namespace Duplicati.Library.Snapshots
{
    /// <summary>
    /// Class that encapsulates USN access to a single volume
    /// </summary>
    public class USNHelper : IDisposable
    {
        /// <summary>
        /// The path this USN points to
        /// </summary>
        private string m_path;

        /// <summary>
        /// The safe filehandle
        /// </summary>
        private SafeFileHandle m_volumeHandle;

        /// <summary>
        /// The current USN journal from the device
        /// </summary>
        private Win32USN.USN_JOURNAL_DATA m_journal;

        /// <summary>
        /// The FileNameReferenceNumber for the root folder
        /// </summary>
        private ulong m_volumeRootFileNameReferenceNumber;

        /// <summary>
        /// This is a cache of all the filesystem entries found on the drive
        /// </summary>
        private List<KeyValuePair<string, Win32USN.USN_RECORD>> m_entryList;

        /// <summary>
        /// Constructs a new USN helper instance
        /// </summary>
        /// <param name="path">The path to the folder to perform USN services</param>
        public USNHelper(string path)
            : this(path, null)
        {
        }


        /// <summary>
        /// Constructs a new USN helper instance
        /// </summary>
        /// <param name="path">The path to the folder to perform USN services</param>
        /// <param name="volumeRoot">The root volume where the USN lookup is performed</param>
        internal USNHelper(string path, string volumeRoot)
        {
            if (Utility.Utility.IsClientLinux)
                throw new Exception(Strings.USNHelper.LinuxNotSupportedError);

            if (!System.IO.Path.IsPathRooted(path))
                throw new Exception(string.Format("Path {0} is not rooted", path));

            m_path = Utility.Utility.AppendDirSeparator(path);

            try
            {
                string devicename = @"\\.\" + System.IO.Path.GetPathRoot(path).TrimEnd('\\');
                if (volumeRoot != null)
                    volumeRoot = volumeRoot.TrimEnd('\\');

                m_volumeHandle = Win32USN.CreateFile(volumeRoot == null ? devicename : volumeRoot, Win32USN.EFileAccess.GenericRead, Win32USN.EFileShare.ReadWrite, IntPtr.Zero, Win32USN.ECreationDisposition.OpenExisting, Win32USN.EFileAttributes.BackupSemantics, IntPtr.Zero);
                if (m_volumeHandle == null || m_volumeHandle.IsInvalid)
                    throw new Win32Exception(Marshal.GetLastWin32Error());

                uint bytesReturned = 0;
                if (!Win32USN.DeviceIoControl(m_volumeHandle, Win32USN.EIOControlCode.FsctlQueryUsnJournal, null, 0, out m_journal, (uint)Marshal.SizeOf(typeof(Win32USN.USN_JOURNAL_DATA)), ref bytesReturned, IntPtr.Zero))
                    throw new Win32Exception(Marshal.GetLastWin32Error());

                Win32USN.BY_HANDLE_FILE_INFORMATION fileInfo;
                using (SafeFileHandle driveHandle = Win32USN.CreateFile(System.IO.Path.GetPathRoot(path), Win32USN.EFileAccess.GenericRead, Win32USN.EFileShare.ReadWrite, IntPtr.Zero, Win32USN.ECreationDisposition.OpenExisting, Win32USN.EFileAttributes.BackupSemantics, IntPtr.Zero))
                    if (!Win32USN.GetFileInformationByHandle(driveHandle, out fileInfo))
                        throw new Win32Exception(Marshal.GetLastWin32Error());

                m_volumeRootFileNameReferenceNumber = ((ulong)fileInfo.FileIndexHigh << 32) | ((ulong)fileInfo.FileIndexLow);
            }
            catch
            {
                if (m_volumeHandle != null)
                {
                    m_volumeHandle.Dispose();
                    m_volumeHandle = null;
                }

                throw;
            }

            if (this.FileSystemEntries.Count == 0)
                throw new Exception(Strings.USNHelper.SafeGuardError);
        }

        /// <summary>
        /// Helper function to support the Duplicati enumeration callback system
        /// </summary>
        /// <param name="rootpath">The root path to look in and use as filter base</param>
        /// <param name="filter">The filter to apply</param>
        /// <param name="callback">The callback function that collects the output</param>
        public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback)
        {
            //Under normal enumeration, the filter will prevent visiting subfolders for excluded folders
            //But when using USN all files/folders are present in the list, so we have to maintain
            // a list of subfolders that are excluded from the set
            System.Text.StringBuilder local_filter = new StringBuilder();
            System.Text.RegularExpressions.Regex excludedFolders = null;

            foreach (KeyValuePair<string, Win32USN.USN_RECORD> r in this.Records)
                if (r.Key.StartsWith(rootpath, Utility.Utility.ClientFilenameStringComparision))
                {
                    bool isFolder = (r.Value.FileAttributes & Win32USN.EFileAttributes.Directory) != 0;

                    if (excludedFolders != null && excludedFolders.Match(r.Key).Success)
                        continue;

                    if (isFolder)
                    {
                        if (filter == null || filter.ShouldInclude(rootpath, Utility.Utility.AppendDirSeparator(r.Key)))
                            callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.Folder);
                        else
                        {
                            if (local_filter.Length != 0)
                                local_filter.Append("|");
                            local_filter.Append("(");
                            local_filter.Append(Utility.FilenameFilter.ConvertGlobbingToRegExp(r.Key + "*"));
                            local_filter.Append(")");

                            excludedFolders = new System.Text.RegularExpressions.Regex(local_filter.ToString());
                        }
                    }
                    else
                    {
                        if (filter == null || filter.ShouldInclude(rootpath, r.Key))
                            callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.File);
                    }
                }
        }<commit_msg>Added a case-sensitivity fix for issue #417<commit_after>#region Disclaimer / License
// Copyright (C) 2011, <NAME>
// http://www.hexad.dk, <EMAIL>
// 
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
// 
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
// 
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
// 
#endregion
using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.Win32.SafeHandles;
using System.Runtime.InteropServices;
using System.ComponentModel;

namespace Duplicati.Library.Snapshots
{
    /// <summary>
    /// Class that encapsulates USN access to a single volume
    /// </summary>
    public class USNHelper : IDisposable
    {
        /// <summary>
        /// The path this USN points to
        /// </summary>
        private string m_path;

        /// <summary>
        /// The safe filehandle
        /// </summary>
        private SafeFileHandle m_volumeHandle;

        /// <summary>
        /// The current USN journal from the device
        /// </summary>
        private Win32USN.USN_JOURNAL_DATA m_journal;

        /// <summary>
        /// The FileNameReferenceNumber for the root folder
        /// </summary>
        private ulong m_volumeRootFileNameReferenceNumber;

        /// <summary>
        /// This is a cache of all the filesystem entries found on the drive
        /// </summary>
        private List<KeyValuePair<string, Win32USN.USN_RECORD>> m_entryList;

        /// <summary>
        /// Constructs a new USN helper instance
        /// </summary>
        /// <param name="path">The path to the folder to perform USN services</param>
        public USNHelper(string path)
            : this(path, null)
        {
        }


        /// <summary>
        /// Constructs a new USN helper instance
        /// </summary>
        /// <param name="path">The path to the folder to perform USN services</param>
        /// <param name="volumeRoot">The root volume where the USN lookup is performed</param>
        internal USNHelper(string path, string volumeRoot)
        {
            if (Utility.Utility.IsClientLinux)
                throw new Exception(Strings.USNHelper.LinuxNotSupportedError);

            if (!System.IO.Path.IsPathRooted(path))
                throw new Exception(string.Format("Path {0} is not rooted", path));

            m_path = Utility.Utility.AppendDirSeparator(path);

            try
            {
                string devicename = @"\\.\" + System.IO.Path.GetPathRoot(path).TrimEnd('\\');
                if (volumeRoot != null)
                    volumeRoot = volumeRoot.TrimEnd('\\');

                m_volumeHandle = Win32USN.CreateFile(volumeRoot == null ? devicename : volumeRoot, Win32USN.EFileAccess.GenericRead, Win32USN.EFileShare.ReadWrite, IntPtr.Zero, Win32USN.ECreationDisposition.OpenExisting, Win32USN.EFileAttributes.BackupSemantics, IntPtr.Zero);
                if (m_volumeHandle == null || m_volumeHandle.IsInvalid)
                    throw new Win32Exception(Marshal.GetLastWin32Error());

                uint bytesReturned = 0;
                if (!Win32USN.DeviceIoControl(m_volumeHandle, Win32USN.EIOControlCode.FsctlQueryUsnJournal, null, 0, out m_journal, (uint)Marshal.SizeOf(typeof(Win32USN.USN_JOURNAL_DATA)), ref bytesReturned, IntPtr.Zero))
                    throw new Win32Exception(Marshal.GetLastWin32Error());

                Win32USN.BY_HANDLE_FILE_INFORMATION fileInfo;
                using (SafeFileHandle driveHandle = Win32USN.CreateFile(System.IO.Path.GetPathRoot(path), Win32USN.EFileAccess.GenericRead, Win32USN.EFileShare.ReadWrite, IntPtr.Zero, Win32USN.ECreationDisposition.OpenExisting, Win32USN.EFileAttributes.BackupSemantics, IntPtr.Zero))
                    if (!Win32USN.GetFileInformationByHandle(driveHandle, out fileInfo))
                        throw new Win32Exception(Marshal.GetLastWin32Error());

                m_volumeRootFileNameReferenceNumber = ((ulong)fileInfo.FileIndexHigh << 32) | ((ulong)fileInfo.FileIndexLow);
            }
            catch
            {
                if (m_volumeHandle != null)
                {
                    m_volumeHandle.Dispose();
                    m_volumeHandle = null;
                }

                throw;
            }

            if (this.FileSystemEntries.Count == 0)
                throw new Exception(Strings.USNHelper.SafeGuardError);
        }

        /// <summary>
        /// Helper function to support the Duplicati enumeration callback system
        /// </summary>
        /// <param name="rootpath">The root path to look in and use as filter base</param>
        /// <param name="filter">The filter to apply</param>
        /// <param name="callback">The callback function that collects the output</param>
        public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback)
        {
            //Under normal enumeration, the filter will prevent visiting subfolders for excluded folders
            //But when using USN all files/folders are present in the list, so we have to maintain
            // a list of subfolders that are excluded from the set
            System.Text.StringBuilder local_filter = new StringBuilder();
            System.Text.RegularExpressions.Regex excludedFolders = null;

            foreach (KeyValuePair<string, Win32USN.USN_RECORD> r in this.Records)
                if (r.Key.StartsWith(rootpath, Utility.Utility.ClientFilenameStringComparision))
                {
                    bool isFolder = (r.Value.FileAttributes & Win32USN.EFileAttributes.Directory) != 0;

                    if (excludedFolders != null && excludedFolders.Match(r.Key).Success)
                        continue;

                    if (isFolder)
                    {
                        if (filter == null || filter.ShouldInclude(rootpath, Utility.Utility.AppendDirSeparator(r.Key)))
                            callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.Folder);
                        else
                        {
                            if (local_filter.Length != 0)
                                local_filter.Append("|");
                            local_filter.Append("(");
                            local_filter.Append(Utility.FilenameFilter.ConvertGlobbingToRegExp(r.Key + "*"));
                            local_filter.Append(")");

                            if (Utility.Utility.IsFSCaseSensitive)
                                excludedFolders = new System.Text.RegularExpressions.Regex(local_filter.ToString());
                            else
                                excludedFolders = new System.Text.RegularExpressions.Regex(local_filter.ToString(), System.Text.RegularExpressions.RegexOptions.IgnoreCase);
                        }
                    }
                    else
                    {
                        if (filter == null || filter.ShouldInclude(rootpath, r.Key))
                            callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.File);
                    }
                }
        }
{'fraction_non_alphanumeric': 0.09451956365536612, 'fraction_numerical': 0.010255405317133712, 'mean_word_length': 1.9867342957471712, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7646095', 'n_tokens_mistral': 4089, 'n_tokens_neox': 3877, 'n_words': 1029}