<commit_before>﻿// Copyright 2015 <NAME> All Rights Reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using mtsuite.shared.Collections;
using mtsuite.shared.Files;
using Microsoft.Win32.SafeHandles;

namespace mtsuite.shared.Win32 {
  public class Win32 {
    private readonly IPool<List<DirectoryEntry>> _entryListPool = new ListPool<DirectoryEntry>();
    private readonly IPool<StringBuffer> _stringBufferPool =
      PoolFactory<StringBuffer>.Create(
        () => new StringBuffer(64),
        x => x.Clear());

    private static string StripPath(string path) {
      if (!PathHelpers.IsPathAbsolute(path))
        return path;
      return PathHelpers.StripLongPathPrefix(path);
    }

    private static string StripPath(IStringSource path) {
      var sb = new StringBuffer(path.Length + 1);
      path.CopyTo(sb);
      return StripPath(sb.Text);
    }

    /// <summary>
    /// Note: For testability, this function should be called through <see cref="IFileSystem"/>.
    /// </summary>
    public FromPool<List<DirectoryEntry>> GetDirectoryEntries(IStringSource path) {
      // Build search pattern (on the stack) as path + "\\*" + '\0'
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        sb.Item.Append(@"\*");

        // Start enumerating files
        WIN32_FIND_DATA data;
        var findHandle = NativeMethods.FindFirstFileEx(
          sb.Item.Data,
          NativeMethods.FINDEX_INFO_LEVELS.FindExInfoBasic,
          out data,
          NativeMethods.FINDEX_SEARCH_OPS.FindExSearchNameMatch,
          IntPtr.Zero,
          NativeMethods.FINDEX_ADDITIONAL_FLAGS.FindFirstExLargeFetch);
        if (findHandle.IsInvalid) {
          var lastWin32Error = Marshal.GetLastWin32Error();
          throw new LastWin32ErrorException(lastWin32Error,
            string.Format("Error enumerating files at \"{0}\"", StripPath(path)));
        }

        using (findHandle) {
          var result = _entryListPool.AllocateFrom();
          try {
            AddResult(ref data, result.Item);
            while (NativeMethods.FindNextFile(findHandle, out data)) {
              AddResult(ref data, result.Item);
            }
            var lastWin32Error = Marshal.GetLastWin32Error();
            if (lastWin32Error != (int)Win32Errors.ERROR_NO_MORE_FILES) {
              throw new LastWin32ErrorException(lastWin32Error,
                string.Format("Error during enumeration of files at \"{0}\"",
                  StripPath(path)));
            }
          } catch {
            result.Dispose();
            throw;
          }
          return result;
        }
      }
    }

    private static void AddResult(ref WIN32_FIND_DATA data, List<DirectoryEntry> entries) {
      var entry = new DirectoryEntry(data.cFileName, data);
      if (SkipSpecialEntry(entry))
        return;

      entries.Add(entry);
    }

    private static bool SkipSpecialEntry(DirectoryEntry entry) {
      return (entry.IsDirectory) && (entry.Name.Equals(".") || entry.Name.Equals(".."));
    }

    public void DeleteFile(IStringSource path) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        if (NativeMethods.DeleteFile(sb.Item.Data))
          return;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error deleting file \"{0}\"", StripPath(path)));
      }
    }

    public void DeleteDirectory(IStringSource path) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        // Note: RemoveDirectory works for empty directories as well
        // as for junction points, even if their target is not empty.
        if (NativeMethods.RemoveDirectory(sb.Item.Data))
          return;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error deleting directory \"{0}\"", StripPath(path)));
      }
    }

    public void SetFileAttributes(IStringSource path, FILE_ATTRIBUTE fileAttributes) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        if (NativeMethods.SetFileAttributes(sb.Item.Data, fileAttributes))
          return;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error setting file attributes on \"{0}\"", StripPath(path)));
      }
    }

    public FILE_ATTRIBUTE GetFileAttributes(IStringSource path) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        var INVALID_FILE_ATTRIBUTES = unchecked((FILE_ATTRIBUTE)(-1));
        var attributes = NativeMethods.GetFileAttributes(sb.Item.Data);
        if (attributes != INVALID_FILE_ATTRIBUTES)
          return attributes;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error getting file attributes for \"{0}\"", StripPath(path)));
      }
    }

    public WIN32_FILE_ATTRIBUTE_DATA GetFileAttributesEx(IStringSource path) {
      WIN32_FILE_ATTRIBUTE_DATA data;
      var lastWin32Error = TryGetFileAttributesEx(path, out data);
      if (lastWin32Error == Win32Errors.ERROR_SUCCESS)
        return data;
      throw new LastWin32ErrorException((int)lastWin32Error,
        string.Format("Error getting extended file attributes for \"{0}\"", StripPath(path)));
    }

    public Win32Errors TryGetFileAttributesEx(IStringSource path, out WIN32_FILE_ATTRIBUTE_DATA data) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        if (NativeMethods.GetFileAttributesEx(sb.Item.Data, 0, out data))
          return Win32Errors.ERROR_SUCCESS;
        return (Win32Errors)Marshal.GetLastWin32Error();
      }
    }

    public void CopyFile(IStringSource sourcePath, IStringSource destinationPath, CopyFileCallback callback) {
      // Note: object lifetime: CopyFileEx terminates within this function call, so
      // is it ok to have [callback] be a local variable.
      using (var source = _stringBufferPool.AllocateFrom())
      using (var destination = _stringBufferPool.AllocateFrom()) {
        sourcePath.CopyTo(source.Item);
        destinationPath.CopyTo(destination.Item);

        Exception error = null;
        NativeMethods.CopyProgressRoutine copyProgress = (size, transferred, streamSize, bytesTransferred, number, reason, file, destinationFile, data) => {
          try {
            callback(transferred, size);
            return NativeMethods.CopyProgressResult.PROGRESS_CONTINUE;
          } catch (Exception e) {
            error = e;
            return NativeMethods.CopyProgressResult.PROGRESS_CANCEL;
          }
        };
        var bCancel = 0;
        var flags = NativeMethods.CopyFileFlags.COPY_FILE_COPY_SYMLINK;
        if (NativeMethods.CopyFileEx(source.Item.Data, destination.Item.Data, copyProgress, IntPtr.Zero, ref bCancel, flags)) {
          return;
        }
        GC.KeepAlive(copyProgress);

        var lastWin32Error = Marshal.GetLastWin32Error();
        if (lastWin32Error == (int)Win32Errors.ERROR_REQUEST_ABORTED && error != null) {
          throw new InvalidOperationException(string.Format("Error copying file from \"{0}\" to \"{1}\"",
            StripPath(sourcePath), StripPath(destinationPath)), error);
        }
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error copying file from \"{0}\" to \"{1}\"",
            StripPath(sourcePath), StripPath(destinationPath)));
      }<commit_msg>Minor code cleanup.<commit_after>﻿// Copyright 2015 <NAME> All Rights Reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using mtsuite.shared.Collections;
using mtsuite.shared.Files;
using Microsoft.Win32.SafeHandles;

namespace mtsuite.shared.Win32 {
  public class Win32 {
    private readonly IPool<List<DirectoryEntry>> _entryListPool = new ListPool<DirectoryEntry>();
    private readonly IPool<StringBuffer> _stringBufferPool =
      PoolFactory<StringBuffer>.Create(
        () => new StringBuffer(64),
        x => x.Clear());

    private static string StripPath(string path) {
      if (!PathHelpers.IsPathAbsolute(path))
        return path;
      return PathHelpers.StripLongPathPrefix(path);
    }

    private static string StripPath(IStringSource path) {
      var sb = new StringBuffer(path.Length + 1);
      path.CopyTo(sb);
      return StripPath(sb.Text);
    }

    /// <summary>
    /// Note: For testability, this function should be called through <see cref="IFileSystem"/>.
    /// </summary>
    public FromPool<List<DirectoryEntry>> GetDirectoryEntries(IStringSource path) {
      // Build search pattern (on the stack) as path + "\\*" + '\0'
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        sb.Item.Append(@"\*");

        // Start enumerating files
        WIN32_FIND_DATA data;
        var findHandle = NativeMethods.FindFirstFileEx(
          sb.Item.Data,
          NativeMethods.FINDEX_INFO_LEVELS.FindExInfoBasic,
          out data,
          NativeMethods.FINDEX_SEARCH_OPS.FindExSearchNameMatch,
          IntPtr.Zero,
          NativeMethods.FINDEX_ADDITIONAL_FLAGS.FindFirstExLargeFetch);
        if (findHandle.IsInvalid) {
          var lastWin32Error = Marshal.GetLastWin32Error();
          throw new LastWin32ErrorException(lastWin32Error,
            string.Format("Error enumerating files at \"{0}\"", StripPath(path)));
        }

        using (findHandle) {
          var result = _entryListPool.AllocateFrom();
          try {
            AddResult(ref data, result.Item);
            while (NativeMethods.FindNextFile(findHandle, out data)) {
              AddResult(ref data, result.Item);
            }
            var lastWin32Error = Marshal.GetLastWin32Error();
            if (lastWin32Error != (int)Win32Errors.ERROR_NO_MORE_FILES) {
              throw new LastWin32ErrorException(lastWin32Error,
                string.Format("Error during enumeration of files at \"{0}\"",
                  StripPath(path)));
            }
          } catch {
            result.Dispose();
            throw;
          }
          return result;
        }
      }
    }

    private static void AddResult(ref WIN32_FIND_DATA data, List<DirectoryEntry> entries) {
      var entry = new DirectoryEntry(data.cFileName, data);
      if (SkipSpecialEntry(entry))
        return;

      entries.Add(entry);
    }

    private static bool SkipSpecialEntry(DirectoryEntry entry) {
      return (entry.IsDirectory) && (entry.Name.Equals(".") || entry.Name.Equals(".."));
    }

    public void DeleteFile(IStringSource path) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        if (NativeMethods.DeleteFile(sb.Item.Data))
          return;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error deleting file \"{0}\"", StripPath(path)));
      }
    }

    public void DeleteDirectory(IStringSource path) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        // Note: RemoveDirectory works for empty directories as well
        // as for junction points, even if their target is not empty.
        if (NativeMethods.RemoveDirectory(sb.Item.Data))
          return;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error deleting directory \"{0}\"", StripPath(path)));
      }
    }

    public void SetFileAttributes(IStringSource path, FILE_ATTRIBUTE fileAttributes) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        if (NativeMethods.SetFileAttributes(sb.Item.Data, fileAttributes))
          return;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error setting file attributes on \"{0}\"", StripPath(path)));
      }
    }

    public FILE_ATTRIBUTE GetFileAttributes(IStringSource path) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        var INVALID_FILE_ATTRIBUTES = unchecked((FILE_ATTRIBUTE)(-1));
        var attributes = NativeMethods.GetFileAttributes(sb.Item.Data);
        if (attributes != INVALID_FILE_ATTRIBUTES)
          return attributes;

        var lastWin32Error = Marshal.GetLastWin32Error();
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error getting file attributes for \"{0}\"", StripPath(path)));
      }
    }

    public WIN32_FILE_ATTRIBUTE_DATA GetFileAttributesEx(IStringSource path) {
      WIN32_FILE_ATTRIBUTE_DATA data;
      var lastWin32Error = TryGetFileAttributesEx(path, out data);
      if (lastWin32Error == Win32Errors.ERROR_SUCCESS)
        return data;
      throw new LastWin32ErrorException((int)lastWin32Error,
        string.Format("Error getting extended file attributes for \"{0}\"", StripPath(path)));
    }

    public Win32Errors TryGetFileAttributesEx(IStringSource path, out WIN32_FILE_ATTRIBUTE_DATA data) {
      using (var sb = _stringBufferPool.AllocateFrom()) {
        path.CopyTo(sb.Item);
        if (NativeMethods.GetFileAttributesEx(sb.Item.Data, 0, out data))
          return Win32Errors.ERROR_SUCCESS;
        return (Win32Errors)Marshal.GetLastWin32Error();
      }
    }

    public void CopyFile(IStringSource sourcePath, IStringSource destinationPath, CopyFileCallback callback) {
      using (var source = _stringBufferPool.AllocateFrom())
      using (var destination = _stringBufferPool.AllocateFrom()) {
        sourcePath.CopyTo(source.Item);
        destinationPath.CopyTo(destination.Item);

        Exception error = null;
        // Note: object lifetime: CopyFileEx terminates within this function
        // call, so is it ok to have "copyProgress" be a local variable.
        // However, we need to call "GC.KeepAlive" to ensure it does not get
        // collected while CopyFileEx is running.
        NativeMethods.CopyProgressRoutine copyProgress = (size, transferred, streamSize, bytesTransferred, number, reason, file, destinationFile, data) => {
          try {
            callback(transferred, size);
            return NativeMethods.CopyProgressResult.PROGRESS_CONTINUE;
          } catch (Exception e) {
            error = e;
            return NativeMethods.CopyProgressResult.PROGRESS_CANCEL;
          }
        };
        var bCancel = 0;
        const NativeMethods.CopyFileFlags flags = NativeMethods.CopyFileFlags.COPY_FILE_COPY_SYMLINK;
        if (NativeMethods.CopyFileEx(source.Item.Data, destination.Item.Data, copyProgress, IntPtr.Zero, ref bCancel, flags)) {
          return;
        }
        GC.KeepAlive(copyProgress);

        var lastWin32Error = Marshal.GetLastWin32Error();
        if (lastWin32Error == (int)Win32Errors.ERROR_REQUEST_ABORTED && error != null) {
          throw new InvalidOperationException(string.Format("Error copying file from \"{0}\" to \"{1}\"",
            StripPath(sourcePath), StripPath(destinationPath)), error);
        }
        throw new LastWin32ErrorException(lastWin32Error,
          string.Format("Error copying file from \"{0}\" to \"{1}\"",
            StripPath(sourcePath), StripPath(destinationPath)));
      }
{'fraction_non_alphanumeric': 0.10092509698597434, 'fraction_numerical': 0.014801551775589377, 'mean_word_length': 2.824697557635243, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7429146', 'n_tokens_mistral': 4748, 'n_tokens_neox': 4550, 'n_words': 1124}