<commit_before>/**
 * @file utils/filesystem.hpp
 * @brief Filesystem utility functions
 * @author <NAME>
 */

#ifndef VIS_UTILS_FILESYSTEM_HPP
#define VIS_UTILS_FILESYSTEM_HPP

#include <boost/filesystem.hpp>
#include <boost/iterator/filter_iterator.hpp>
#include <functional>
#include <string>
#include <vector>

namespace vis {

/// Match image file paths.
struct has_image_extension {
    bool operator()(const boost::filesystem::path& p) {
        const boost::filesystem::path ext = p.extension();
        return ext == ".png" || ext == ".jpg";
    }
};

/// Get image files in given directory.
std::vector<boost::filesystem::path>
getImageFiles(const boost::filesystem::path& dir) {
    std::vector<boost::filesystem::path> files;

    boost::filesystem::directory_iterator it = boost::filesystem::directory_iterator(dir),
                           end = boost::filesystem::directory_iterator();

    std::copy(boost::make_filter_iterator<has_image_extension>(it, end),
              boost::make_filter_iterator<has_image_extension>(end, end),
              std::back_inserter(files));

    return files;
}

// TODO unit test for this
void path2string(const std::vector<boost::filesystem::path>& paths, std::vector<std::string>& names) {
    std::transform(paths.begin(), paths.end(), std::back_inserter(names),
                   std::mem_fun_ref<std::string,boost::filesystem::path>(&boost::filesystem::path::string));
}

} /* namespace vis */

#endif /* VIS_UTILS_FILESYSTEM_HPP */
<commit_msg>Minor refactoring<commit_after>/**
 * @file utils/filesystem.hpp
 * @brief Filesystem utility functions
 * @author <NAME>
 */

#ifndef VIS_UTILS_FILESYSTEM_HPP
#define VIS_UTILS_FILESYSTEM_HPP

#include <boost/filesystem.hpp>
#include <boost/iterator/filter_iterator.hpp>
#include <functional>
#include <string>
#include <vector>

namespace vis {

/// Match image file paths.
struct has_image_extension {
    bool operator()(const boost::filesystem::path& p) {
        const boost::filesystem::path ext = p.extension();
        return ext == ".png" || ext == ".jpg";
    }
};

/// Get image files in given directory.
std::vector<boost::filesystem::path>
getImageFiles(const boost::filesystem::path& dir) {
    std::vector<boost::filesystem::path> files;

    boost::filesystem::directory_iterator
            it = boost::filesystem::directory_iterator(dir),
            end = boost::filesystem::directory_iterator();

    std::copy(boost::make_filter_iterator<has_image_extension>(it, end),
              boost::make_filter_iterator<has_image_extension>(end, end),
              std::back_inserter(files));

    return files;
}

// TODO unit test for this
void
path2string(const std::vector<boost::filesystem::path>& paths,
        std::vector<std::string>& names) {
    using boost::filesystem::path;
    std::transform(paths.begin(), paths.end(),
                   std::back_inserter(names),
                   std::mem_fun_ref<std::string, path>(&path::string));
}

} /* namespace vis */

#endif /* VIS_UTILS_FILESYSTEM_HPP */

{'fraction_non_alphanumeric': 0.1492094861660079, 'fraction_numerical': 0.0006587615283267457, 'mean_word_length': 4.20926243567753, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5377486', 'n_tokens_mistral': 938, 'n_tokens_neox': 928, 'n_words': 178}