<commit_before>#define __STDC_WANT_LIB_EXT1__
#include "File.h"
#include "Exception.h"

/////////////////////////////////////////////////////////////////////////////
// System-specific functions
/////////////////////////////////////////////////////////////////////////////
#ifdef _WIN32
#include <Windows.h>
#include <io.h>
#include <stdio.h>
#include <FileAPI.h>
#include <corecrt_io.h>

bool joedb::File::lock_file()
{
 HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file));
 return LockFile(hFile, 0, 0, 1, 0) == TRUE;
}

void joedb::File::sync()
{
 _commit(_fileno(file));
}

#elif defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
#include <sys/file.h>
#include <unistd.h>

bool joedb::File::lock_file()
{
 return flock(fileno(file), LOCK_EX | LOCK_NB) == 0;
}

void joedb::File::sync()
{
 fsync(fileno(file));
}

#else
#pragma message("warning: unknown system: no lock, no sync")

bool joedb::File::lock_file()
{
 return true;
}

void joedb::File::sync()
{
}

#endif

/////////////////////////////////////////////////////////////////////////////
bool joedb::File::try_open(const char *file_name, Open_Mode new_mode)
/////////////////////////////////////////////////////////////////////////////
{
 static const char *mode_string[3] = {"rb", "r+b", "w+b"};
 mode = new_mode;
 fopen_s(&file, file_name, mode_string[static_cast<size_t>(mode)]);
 return file != 0;
}

/////////////////////////////////////////////////////////////////////////////
joedb::File::File(const char *file_name, Open_Mode new_mode)
/////////////////////////////////////////////////////////////////////////////
{
 if (new_mode == Open_Mode::write_existing_or_create_new)
 {
  try_open(file_name, Open_Mode::write_existing) ||
  try_open(file_name, Open_Mode::create_new);
 }
 else if (new_mode == Open_Mode::create_new)
 {
  if (try_open(file_name, Open_Mode::read_existing))
  {
   close_file();
   throw Exception("File already exists: " + std::string(file_name));
  }
  else
   try_open(file_name, Open_Mode::create_new);
 }
 else
  try_open(file_name, new_mode);
<commit_msg>Revert "silence msvc warning" (gcc does not have fopen_s)<commit_after>#include "File.h"
#include "Exception.h"

/////////////////////////////////////////////////////////////////////////////
// System-specific functions
/////////////////////////////////////////////////////////////////////////////
#ifdef _WIN32
#include <Windows.h>
#include <io.h>
#include <stdio.h>
#include <FileAPI.h>
#include <corecrt_io.h>

bool joedb::File::lock_file()
{
 HANDLE hFile = (HANDLE)_get_osfhandle(_fileno(file));
 return LockFile(hFile, 0, 0, 1, 0) == TRUE;
}

void joedb::File::sync()
{
 _commit(_fileno(file));
}

#elif defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
#include <sys/file.h>
#include <unistd.h>

bool joedb::File::lock_file()
{
 return flock(fileno(file), LOCK_EX | LOCK_NB) == 0;
}

void joedb::File::sync()
{
 fsync(fileno(file));
}

#else
#pragma message("warning: unknown system: no lock, no sync")

bool joedb::File::lock_file()
{
 return true;
}

void joedb::File::sync()
{
}

#endif

/////////////////////////////////////////////////////////////////////////////
bool joedb::File::try_open(const char *file_name, Open_Mode new_mode)
/////////////////////////////////////////////////////////////////////////////
{
 static const char *mode_string[3] = {"rb", "r+b", "w+b"};
 mode = new_mode;
 file = std::fopen(file_name, mode_string[static_cast<size_t>(mode)]);
 return file != 0;
}

/////////////////////////////////////////////////////////////////////////////
joedb::File::File(const char *file_name, Open_Mode new_mode)
/////////////////////////////////////////////////////////////////////////////
{
 if (new_mode == Open_Mode::write_existing_or_create_new)
 {
  try_open(file_name, Open_Mode::write_existing) ||
  try_open(file_name, Open_Mode::create_new);
 }
 else if (new_mode == Open_Mode::create_new)
 {
  if (try_open(file_name, Open_Mode::read_existing))
  {
   close_file();
   throw Exception("File already exists: " + std::string(file_name));
  }
  else
   try_open(file_name, Open_Mode::create_new);
 }
 else
  try_open(file_name, new_mode);

{'fraction_non_alphanumeric': 0.35243139142994706, 'fraction_numerical': 0.0045739046701974, 'mean_word_length': 1.9806312769010044, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6397402', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1298, 'n_words': 254}