<commit_before>
#include <cstring>
#ifdef _WIN32
#include <windows.h>
#else
#include <cstdio>
#include <cerrno>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#endif

#include "Assert.h"
#include "File.h"
#include "String.h"

File::File()
{
#ifdef _WIN32
  ASSERT(sizeof(void*) >= sizeof(HANDLE));
  fp = INVALID_HANDLE_VALUE;
#else
  fp = 0;
#endif
}

File::~File()
{
  close();
}

void File::close()
{
#ifdef _WIN32
  if(fp != INVALID_HANDLE_VALUE)
  {
    CloseHandle((HANDLE)fp);
    fp = INVALID_HANDLE_VALUE;
  }
#else
  if(fp)
  {
    fclose((FILE*)fp);
    fp = 0;
  }
#endif
}

bool File::unlink(const String& file)
{
#ifdef _WIN32
  if(!DeleteFile(file.getData()))
    return false;
#else
  if(::unlink(file.getData()) != 0)
    return false;
#endif
  return true;
}

bool File::open(const String& file, Flags flags)
{
#ifdef _WIN32
  if(fp != INVALID_HANDLE_VALUE)
  {
    SetLastError(ERROR_INVALID_HANDLE);
    return false;
  }
  DWORD desiredAccess = 0, creationDisposition = 0;
  if(flags & writeFlag)
  {
    desiredAccess |= GENERIC_WRITE;
    creationDisposition |= CREATE_ALWAYS;
  }
  if(flags & readFlag)
  {
    desiredAccess |= GENERIC_READ;
    if(!(flags & writeFlag))
      creationDisposition |= OPEN_EXISTING;
  }
  fp = CreateFileA(file.getData(), desiredAccess, FILE_SHARE_READ, NULL, creationDisposition, FILE_ATTRIBUTE_NORMAL, NULL);
  if(fp == INVALID_HANDLE_VALUE)
    return false;
#else
  if(fp)
    return false;
  const char* mode = (flags & (writeFlag | readFlag)) == (writeFlag | readFlag) ? "w+" : (flags & writeFlag ? "w" : "r");<commit_msg>Support long path names on windows<commit_after>
#include <cstring>
#ifdef _WIN32
#include <windows.h>
#include <malloc.h>
#else
#include <cstdio>
#include <cerrno>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#endif

#include "Assert.h"
#include "File.h"
#include "String.h"

File::File()
{
#ifdef _WIN32
  ASSERT(sizeof(void*) >= sizeof(HANDLE));
  fp = INVALID_HANDLE_VALUE;
#else
  fp = 0;
#endif
}

File::~File()
{
  close();
}

void File::close()
{
#ifdef _WIN32
  if(fp != INVALID_HANDLE_VALUE)
  {
    CloseHandle((HANDLE)fp);
    fp = INVALID_HANDLE_VALUE;
  }
#else
  if(fp)
  {
    fclose((FILE*)fp);
    fp = 0;
  }
#endif
}

bool File::unlink(const String& file)
{
#ifdef _WIN32
  if(!DeleteFile(file.getData()))
    return false;
#else
  if(::unlink(file.getData()) != 0)
    return false;
#endif
  return true;
}

bool File::open(const String& file, Flags flags)
{
#ifdef _WIN32
  if(fp != INVALID_HANDLE_VALUE)
  {
    SetLastError(ERROR_INVALID_HANDLE);
    return false;
  }
  DWORD desiredAccess = 0, creationDisposition = 0;
  if(flags & writeFlag)
  {
    desiredAccess |= GENERIC_WRITE;
    creationDisposition |= CREATE_ALWAYS;
  }
  if(flags & readFlag)
  {
    desiredAccess |= GENERIC_READ;
    if(!(flags & writeFlag))
      creationDisposition |= OPEN_EXISTING;
  }
  fp = CreateFileA(file.getData(), desiredAccess, FILE_SHARE_READ, NULL, creationDisposition, FILE_ATTRIBUTE_NORMAL, NULL);
  if (fp == INVALID_HANDLE_VALUE)
  {
      if (GetLastError() == ERROR_PATH_NOT_FOUND) // path too long? try unicode stuff
      {
          String absFile = file;
          if (!File::isPathAbsolute(file))
          {
              char curDir[MAX_PATH];;
              DWORD curDirLen;
              if ((curDirLen = GetCurrentDirectoryA(MAX_PATH, curDir)) == 0)
              {
                  SetLastError(ERROR_PATH_NOT_FOUND);
                  return false;
              }
              absFile.clear();
              absFile.setCapacity(curDirLen + 1 + file.getLength());
              absFile.append(curDir, curDirLen);
              absFile.append('\\');
              absFile.append(file);
              absFile = File::simplifyPath(absFile);
          }

          int wFileLen = MultiByteToWideChar(CP_ACP, 0, absFile.getData(), absFile.getLength(), 0, 0);
          int wlen = 4 + wFileLen + 1;
          WCHAR* path = (WCHAR*)alloca(wlen * sizeof(WCHAR));
          memcpy(path, L"\\\\?\\", 4 * sizeof(WCHAR));
          MultiByteToWideChar(CP_ACP, 0, absFile.getData(), absFile.getLength(), path + 4, wFileLen + 1);
          path[4 + wFileLen] = L'\0';
          for (WCHAR* p = path + 4; *p; ++p)
            if (*p == L'/')
                *p = L'\\';
          fp = CreateFileW(path, desiredAccess, FILE_SHARE_READ, NULL, creationDisposition, FILE_ATTRIBUTE_NORMAL, NULL);
          if (fp == INVALID_HANDLE_VALUE)
              return false;
      }
      else
          return false;
  }
#else
  if(fp)
    return false;
  const char* mode = (flags & (writeFlag | readFlag)) == (writeFlag | readFlag) ? "w+" : (flags & writeFlag ? "w" : "r");
{'fraction_non_alphanumeric': 0.14528462192013594, 'fraction_numerical': 0.009345794392523364, 'mean_word_length': 2.983925549915398, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5739607', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1671, 'n_words': 363}