<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "disk_interface.h"

#include <algorithm>

#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>

#ifdef _WIN32
#include <windows.h>
#include <direct.h>  // _mkdir
#endif

#include "util.h"

namespace {

string DirName(const string& path) {
#ifdef _WIN32
  const char kPathSeparators[] = "\\/";
#else
  const char kPathSeparators[] = "/";
#endif
  string::size_type slash_pos = path.find_last_of(kPathSeparators);
  if (slash_pos == string::npos)
    return string();  // Nothing to do.
  const char* const kEnd = kPathSeparators + strlen(kPathSeparators);
  while (slash_pos > 0 &&
         std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd)
    --slash_pos;
  return path.substr(0, slash_pos);
}

int MakeDir(const string& path) {
#ifdef _WIN32
  return _mkdir(path.c_str());
#else
  return mkdir(path.c_str(), 0777);
#endif
}

#ifdef _WIN32
TimeStamp TimeStampFromFileTime(const FILETIME& filetime) {
  // FILETIME is in 100-nanosecond increments since the Windows epoch.
  // We don't much care about epoch correctness but we do want the
  // resulting value to fit in an integer.
  uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
    ((uint64_t)filetime.dwLowDateTime);
  mtime /= 1000000000LL / 100; // 100ns -> s.
  mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
  return (TimeStamp)mtime;
}

TimeStamp StatSingleFile(const string& path, bool quiet) {
  WIN32_FILE_ATTRIBUTE_DATA attrs;
  if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
    DWORD err = GetLastError();
    if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
      return 0;
    if (!quiet) {
      Error("GetFileAttributesEx(%s): %s", path.c_str(),
            GetLastErrorString().c_str());
    }
    return -1;
  }
  return TimeStampFromFileTime(attrs.ftLastWriteTime);
}

bool IsWindows7OrLater() {
  OSVERSIONINFO version_info = { sizeof(version_info) };
  if (!GetVersionEx(&version_info))
    Fatal("GetVersionEx: %s", GetLastErrorString().c_str());
  return version_info.dwMajorVersion > 6 ||
         version_info.dwMajorVersion == 6 && version_info.dwMinorVersion >= 1;
}

bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
                       bool quiet) {
  // FindExInfoBasic is 30% faster than FindExInfoStandard.
  static bool can_use_basic_info = IsWindows7OrLater();
  FINDEX_INFO_LEVELS level =
      can_use_basic_info ? FindExInfoBasic : FindExInfoStandard;
  WIN32_FIND_DATAA ffd;
  HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd,
                                        FindExSearchNameMatch, NULL, 0);

  if (find_handle == INVALID_HANDLE_VALUE) {
    DWORD err = GetLastError();<commit_msg>suppress warning on win8.1 sdk<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "disk_interface.h"

#include <algorithm>

#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>

#ifdef _WIN32
#include <windows.h>
#include <direct.h>  // _mkdir
#endif

#include "util.h"

namespace {

string DirName(const string& path) {
#ifdef _WIN32
  const char kPathSeparators[] = "\\/";
#else
  const char kPathSeparators[] = "/";
#endif
  string::size_type slash_pos = path.find_last_of(kPathSeparators);
  if (slash_pos == string::npos)
    return string();  // Nothing to do.
  const char* const kEnd = kPathSeparators + strlen(kPathSeparators);
  while (slash_pos > 0 &&
         std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd)
    --slash_pos;
  return path.substr(0, slash_pos);
}

int MakeDir(const string& path) {
#ifdef _WIN32
  return _mkdir(path.c_str());
#else
  return mkdir(path.c_str(), 0777);
#endif
}

#ifdef _WIN32
TimeStamp TimeStampFromFileTime(const FILETIME& filetime) {
  // FILETIME is in 100-nanosecond increments since the Windows epoch.
  // We don't much care about epoch correctness but we do want the
  // resulting value to fit in an integer.
  uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
    ((uint64_t)filetime.dwLowDateTime);
  mtime /= 1000000000LL / 100; // 100ns -> s.
  mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
  return (TimeStamp)mtime;
}

TimeStamp StatSingleFile(const string& path, bool quiet) {
  WIN32_FILE_ATTRIBUTE_DATA attrs;
  if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
    DWORD err = GetLastError();
    if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
      return 0;
    if (!quiet) {
      Error("GetFileAttributesEx(%s): %s", path.c_str(),
            GetLastErrorString().c_str());
    }
    return -1;
  }
  return TimeStampFromFileTime(attrs.ftLastWriteTime);
}

#pragma warning(push)
#pragma warning(disable: 4996)  // GetVersionExA is deprecated post SDK 8.1.
bool IsWindows7OrLater() {
  OSVERSIONINFO version_info = { sizeof(version_info) };
  if (!GetVersionEx(&version_info))
    Fatal("GetVersionEx: %s", GetLastErrorString().c_str());
  return version_info.dwMajorVersion > 6 ||
         version_info.dwMajorVersion == 6 && version_info.dwMinorVersion >= 1;
}
#pragma warning(pop)

bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
                       bool quiet) {
  // FindExInfoBasic is 30% faster than FindExInfoStandard.
  static bool can_use_basic_info = IsWindows7OrLater();
  FINDEX_INFO_LEVELS level =
      can_use_basic_info ? FindExInfoBasic : FindExInfoStandard;
  WIN32_FIND_DATAA ffd;
  HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd,
                                        FindExSearchNameMatch, NULL, 0);

  if (find_handle == INVALID_HANDLE_VALUE) {
    DWORD err = GetLastError();
{'fraction_non_alphanumeric': 0.11912631125161662, 'fraction_numerical': 0.025865785313981895, 'mean_word_length': 3.7443762781186094, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3999361', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2225, 'n_words': 640}