<commit_before>// Copyright (C) 2018 <NAME> <<EMAIL>>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#pragma once

#include <primitives/enums.h>
#include <primitives/string.h>

#include <filesystem>
#include <unordered_map>
#include <unordered_set>

namespace fs = std::filesystem;
using error_code = std::error_code;

using path = fs::path;

#ifdef _WIN32
// on win and C++20 we can use std::u8string for path strings
// to convert paths in both directions safely
#define PRIMITIVES_FS_USE_UTF8_PATH_STRINGS 1
#else
#define PRIMITIVES_FS_USE_UTF8_PATH_STRINGS 0
#endif

#if PRIMITIVES_FS_USE_UTF8_PATH_STRINGS
// on win and C++20 we can use std::u8string for path strings
// to convert paths in both directions safely
using path_u8string = std::u8string;
using path_char_t = char8_t;
#else
using path_u8string = String;
using path_char_t = char;
#endif

using FilesSorted = std::set<path>;
using FilesOrdered = std::vector<path>;
using Files = std::unordered_set<path>;
using FilesMap = std::unordered_map<path, path>;

PRIMITIVES_FILESYSTEM_API
path get_home_directory();

PRIMITIVES_FILESYSTEM_API
path current_thread_path(const path &p = path());

PRIMITIVES_FILESYSTEM_API
String read_file(const path &p, uintmax_t offset = 0, uintmax_t count = UINTMAX_MAX);

PRIMITIVES_FILESYSTEM_API
String read_file_without_bom(const path &p, uintmax_t offset_after_bom = 0, uintmax_t count = UINTMAX_MAX);

PRIMITIVES_FILESYSTEM_API
void write_file(const path &p, const String &);

PRIMITIVES_FILESYSTEM_API
void write_file(const path &p, const std::vector<uint8_t> &);

PRIMITIVES_FILESYSTEM_API
void write_file_if_different(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
void write_file_if_not_exists(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
void prepend_file(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
void append_file(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
Strings read_lines(const path &p);

PRIMITIVES_FILESYSTEM_API
void remove_all_from_dir(const path &dir);

/// change all '\\' into '/', make drive letter uppercase
PRIMITIVES_FILESYSTEM_API
path normalize_path(const path &p);

/// change all '/' into '\\', make drive letter uppercase
PRIMITIVES_FILESYSTEM_API
path normalize_path_windows(const path &p);

/// returns utf-8 string
PRIMITIVES_FILESYSTEM_API
path_u8string to_path_string(const path &p);

/// returns utf-8 string in std::string
PRIMITIVES_FILESYSTEM_API
String to_printable_string(const path &p);

// checks for real file
PRIMITIVES_FILESYSTEM_API
bool is_under_root(path existing_path, const path &root_dir);

// checks path that may not exist
PRIMITIVES_FILESYSTEM_API
bool is_under_root_by_prefix_path(const path &p, const path &root_dir);

PRIMITIVES_FILESYSTEM_API
void copy_dir(const path &source, const path &destination);

PRIMITIVES_FILESYSTEM_API
void remove_files(const Files &files);

PRIMITIVES_FILESYSTEM_API
void remove_files_like(const path &dir, const String &regex, bool recursive = true);

PRIMITIVES_FILESYSTEM_API
void remove_files_like(const Files &files, const String &regex);

PRIMITIVES_FILESYSTEM_API
Files enumerate_files(const path &dir, bool recursive = true);

PRIMITIVES_FILESYSTEM_API
Files enumerate_files_like(const path &dir, const String &regex, bool recursive = true);

PRIMITIVES_FILESYSTEM_API
Files enumerate_files_like(const Files &files, const String &regex);

PRIMITIVES_FILESYSTEM_API
Files filter_files_like(const Files &files, const String &regex);

PRIMITIVES_FILESYSTEM_API
bool compare_files(const path &fn1, const path &fn2);

PRIMITIVES_FILESYSTEM_API
bool compare_dirs(const path &dir1, const path &dir2);

PRIMITIVES_FILESYSTEM_API
path unique_path(const path &p = "%%%%-%%%%-%%%%-%%%%");

PRIMITIVES_FILESYSTEM_API
time_t file_time_type2time_t(fs::file_time_type);

PRIMITIVES_FILESYSTEM_API
fs::file_time_type time_t2file_time_type(time_t);

namespace primitives::filesystem
{

PRIMITIVES_FILESYSTEM_API
void remove_file(const path &p);

PRIMITIVES_FILESYSTEM_API
path canonical(const path &p);

}

#if defined(_MSC_VER) && _MSC_VER < 1932 || __APPLE__ || defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11<commit_msg>Update filesystem.h<commit_after>// Copyright (C) 2018 <NAME> <<EMAIL>>
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#pragma once

#include <primitives/enums.h>
#include <primitives/string.h>

#include <filesystem>
#include <unordered_map>
#include <unordered_set>

namespace fs = std::filesystem;
using error_code = std::error_code;

using path = fs::path;

#ifdef _WIN32
// on win and C++20 we can use std::u8string for path strings
// to convert paths in both directions safely
#define PRIMITIVES_FS_USE_UTF8_PATH_STRINGS 1
#else
#define PRIMITIVES_FS_USE_UTF8_PATH_STRINGS 0
#endif

#if PRIMITIVES_FS_USE_UTF8_PATH_STRINGS
// on win and C++20 we can use std::u8string for path strings
// to convert paths in both directions safely
using path_u8string = std::u8string;
using path_char_t = char8_t;
#else
using path_u8string = String;
using path_char_t = char;
#endif

using FilesSorted = std::set<path>;
using FilesOrdered = std::vector<path>;
using Files = std::unordered_set<path>;
using FilesMap = std::unordered_map<path, path>;

PRIMITIVES_FILESYSTEM_API
path get_home_directory();

PRIMITIVES_FILESYSTEM_API
path current_thread_path(const path &p = path());

PRIMITIVES_FILESYSTEM_API
String read_file(const path &p, uintmax_t offset = 0, uintmax_t count = UINTMAX_MAX);

PRIMITIVES_FILESYSTEM_API
String read_file_without_bom(const path &p, uintmax_t offset_after_bom = 0, uintmax_t count = UINTMAX_MAX);

PRIMITIVES_FILESYSTEM_API
void write_file(const path &p, const String &);

PRIMITIVES_FILESYSTEM_API
void write_file(const path &p, const std::vector<uint8_t> &);

PRIMITIVES_FILESYSTEM_API
void write_file_if_different(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
void write_file_if_not_exists(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
void prepend_file(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
void append_file(const path &p, const String &s);

PRIMITIVES_FILESYSTEM_API
Strings read_lines(const path &p);

PRIMITIVES_FILESYSTEM_API
void remove_all_from_dir(const path &dir);

/// change all '\\' into '/', make drive letter uppercase
PRIMITIVES_FILESYSTEM_API
path normalize_path(const path &p);

/// change all '/' into '\\', make drive letter uppercase
PRIMITIVES_FILESYSTEM_API
path normalize_path_windows(const path &p);

/// returns utf-8 string
PRIMITIVES_FILESYSTEM_API
path_u8string to_path_string(const path &p);

/// returns utf-8 string in std::string
PRIMITIVES_FILESYSTEM_API
String to_printable_string(const path &p);

// checks for real file
PRIMITIVES_FILESYSTEM_API
bool is_under_root(path existing_path, const path &root_dir);

// checks path that may not exist
PRIMITIVES_FILESYSTEM_API
bool is_under_root_by_prefix_path(const path &p, const path &root_dir);

PRIMITIVES_FILESYSTEM_API
void copy_dir(const path &source, const path &destination);

PRIMITIVES_FILESYSTEM_API
void remove_files(const Files &files);

PRIMITIVES_FILESYSTEM_API
void remove_files_like(const path &dir, const String &regex, bool recursive = true);

PRIMITIVES_FILESYSTEM_API
void remove_files_like(const Files &files, const String &regex);

PRIMITIVES_FILESYSTEM_API
Files enumerate_files(const path &dir, bool recursive = true);

PRIMITIVES_FILESYSTEM_API
Files enumerate_files_like(const path &dir, const String &regex, bool recursive = true);

PRIMITIVES_FILESYSTEM_API
Files enumerate_files_like(const Files &files, const String &regex);

PRIMITIVES_FILESYSTEM_API
Files filter_files_like(const Files &files, const String &regex);

PRIMITIVES_FILESYSTEM_API
bool compare_files(const path &fn1, const path &fn2);

PRIMITIVES_FILESYSTEM_API
bool compare_dirs(const path &dir1, const path &dir2);

PRIMITIVES_FILESYSTEM_API
path unique_path(const path &p = "%%%%-%%%%-%%%%-%%%%");

PRIMITIVES_FILESYSTEM_API
time_t file_time_type2time_t(fs::file_time_type);

PRIMITIVES_FILESYSTEM_API
fs::file_time_type time_t2file_time_type(time_t);

namespace primitives::filesystem
{

PRIMITIVES_FILESYSTEM_API
void remove_file(const path &p);

PRIMITIVES_FILESYSTEM_API
path canonical(const path &p);

}

#if defined(_MSC_VER) && _MSC_VER < 1932 || __APPLE__ || defined(__GLIBCXX__) && __GLIBCXX__ < 20220421 // __GLIBCXX__ < 11.3
{'fraction_non_alphanumeric': 0.09012231710131549, 'fraction_numerical': 0.010962381721670899, 'mean_word_length': 6.724598930481283, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '126146', 'n_tokens_mistral': 3053, 'n_tokens_neox': 3111, 'n_words': 810}