<commit_before>#define CATCH_CONFIG_MAIN
#include "catch.hpp"

#include <mapnik/map.hpp>
#include <mapnik/datasource.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/geometry_types.hpp>
#include <mapnik/geometry_type.hpp>
#include <mapnik/expression.hpp>
#include <mapnik/expression_evaluator.hpp>
#include <mapnik/debug.hpp>
#include <mapnik/util/fs.hpp>
#include <boost/filesystem.hpp>
#include <boost/range/iterator_range_core.hpp>
#include <boost/format.hpp>
#include <boost/optional/optional_io.hpp>

#include <iostream>

namespace bfs = boost::filesystem;

namespace {
void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
{
    for (auto const &entry : boost::make_iterator_range(
             bfs::directory_iterator(dir), bfs::directory_iterator()))
    {
        auto path = entry.path();
        if (path.extension().native() == ".csv")
        {
            csv_files.emplace_back(path);
        }
    }
}

mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
{
    mapnik::parameters params;
    params["type"] = std::string("csv");
    params["file"] = file_name;
    params["strict"] = mapnik::value_bool(strict);
    auto ds = mapnik::datasource_cache::instance().create(params);
    // require a non-null pointer returned
    REQUIRE(ds != nullptr);
    return ds;
}

void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
                         std::initializer_list<std::string> const &names)
{
    REQUIRE(fields.size() == names.size());
    auto itr_a = fields.begin();
    auto const end_a = fields.end();
    auto itr_b = names.begin();
    for (; itr_a != end_a; ++itr_a, ++itr_b)
    {
        CHECK(itr_a->get_name() == *itr_b);
    }
}

void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
                         std::initializer_list<mapnik::eAttributeType> const &types) {
    REQUIRE(fields.size() == types.size());
    auto itr_a = fields.begin();
    auto const end_a = fields.end();
    auto itr_b = types.begin();
    for (; itr_a != end_a; ++itr_a, ++itr_b) {
        CHECK(itr_a->get_type() == *itr_b);
    }
}

mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
    auto fields = ds->get_descriptor().get_descriptors();
    mapnik::query query(ds->envelope());
    for (auto const &field : fields) {
        query.add_property_name(field.get_name());
    }
    return ds->features(query);
}

std::size_t count_features(mapnik::featureset_ptr features) {
    std::size_t count = 0;
    while (features->next()) {
        ++count;
    }
    return count;
}

using attr = std::tuple<std::string, mapnik::value>;
void require_attributes(mapnik::feature_ptr feature,
                        std::initializer_list<attr> const &attrs) {
    REQUIRE(bool(feature));
    for (auto const &kv : attrs) {
        REQUIRE(feature->has_key(std::get<0>(kv)));
        CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
    }
}

namespace detail {
struct feature_count {
    template <typename T>
    std::size_t operator()(T const &geom) const {
        return mapnik::util::apply_visitor(*this, geom);
    }

    std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
        return 0;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::point<T> const &) const {
        return 1;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
        return 1;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
        return 1;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
        return mp.size();
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
        return mls.size();
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
        return mp.size();
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
        std::size_t sum = 0;
        for (auto const &geom : col) {
            sum += operator()(geom);
        }
        return sum;
    }
};
} // namespace detail

template <typename T>
std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
    return detail::feature_count()(g);
}

void require_geometry(mapnik::feature_ptr feature,
                      std::size_t num_parts,
                      mapnik::geometry::geometry_types type) {
    REQUIRE(bool(feature));
    CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
    CHECK(feature_count(feature->get_geometry()) == num_parts);
}
} // anonymous namespace

static const std::string csv_plugin("./plugins/input/csv.input");

const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);

TEST_CASE("csv") {

    if (mapnik::util::exists(csv_plugin))
    {
        REQUIRE(registered);
        // make the tests silent since we intentially test error conditions that are noisy
        auto const severity = mapnik::logger::instance().get_severity();
        mapnik::logger::instance().set_severity(mapnik::logger::none);

        // check the CSV datasource is loaded
        const std::vector<std::string> plugin_names =
            mapnik::datasource_cache::instance().plugin_names();
        const bool have_csv_plugin =
            std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();

        SECTION("broken files") {
            if (have_csv_plugin) {
                std::vector<bfs::path> broken;
                add_csv_files("test/data/csv/fails", broken);
                add_csv_files("test/data/csv/warns", broken);
                broken.emplace_back("test/data/csv/fails/does_not_exist.csv");

                for (auto const &path : broken)
                {
                    REQUIRE_THROWS(get_csv_ds(path.native()));<commit_msg>log csv filepath when test fails [skip ci]<commit_after>#define CATCH_CONFIG_MAIN
#include "catch.hpp"

#include <mapnik/map.hpp>
#include <mapnik/datasource.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/geometry.hpp>
#include <mapnik/geometry_types.hpp>
#include <mapnik/geometry_type.hpp>
#include <mapnik/expression.hpp>
#include <mapnik/expression_evaluator.hpp>
#include <mapnik/debug.hpp>
#include <mapnik/util/fs.hpp>
#include <boost/filesystem.hpp>
#include <boost/range/iterator_range_core.hpp>
#include <boost/format.hpp>
#include <boost/optional/optional_io.hpp>

#include <iostream>

namespace bfs = boost::filesystem;

namespace {
void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
{
    for (auto const &entry : boost::make_iterator_range(
             bfs::directory_iterator(dir), bfs::directory_iterator()))
    {
        auto path = entry.path();
        if (path.extension().native() == ".csv")
        {
            csv_files.emplace_back(path);
        }
    }
}

mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
{
    mapnik::parameters params;
    params["type"] = std::string("csv");
    params["file"] = file_name;
    params["strict"] = mapnik::value_bool(strict);
    auto ds = mapnik::datasource_cache::instance().create(params);
    // require a non-null pointer returned
    REQUIRE(ds != nullptr);
    return ds;
}

void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
                         std::initializer_list<std::string> const &names)
{
    REQUIRE(fields.size() == names.size());
    auto itr_a = fields.begin();
    auto const end_a = fields.end();
    auto itr_b = names.begin();
    for (; itr_a != end_a; ++itr_a, ++itr_b)
    {
        CHECK(itr_a->get_name() == *itr_b);
    }
}

void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
                         std::initializer_list<mapnik::eAttributeType> const &types) {
    REQUIRE(fields.size() == types.size());
    auto itr_a = fields.begin();
    auto const end_a = fields.end();
    auto itr_b = types.begin();
    for (; itr_a != end_a; ++itr_a, ++itr_b) {
        CHECK(itr_a->get_type() == *itr_b);
    }
}

mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
    auto fields = ds->get_descriptor().get_descriptors();
    mapnik::query query(ds->envelope());
    for (auto const &field : fields) {
        query.add_property_name(field.get_name());
    }
    return ds->features(query);
}

std::size_t count_features(mapnik::featureset_ptr features) {
    std::size_t count = 0;
    while (features->next()) {
        ++count;
    }
    return count;
}

using attr = std::tuple<std::string, mapnik::value>;
void require_attributes(mapnik::feature_ptr feature,
                        std::initializer_list<attr> const &attrs) {
    REQUIRE(bool(feature));
    for (auto const &kv : attrs) {
        REQUIRE(feature->has_key(std::get<0>(kv)));
        CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
    }
}

namespace detail {
struct feature_count {
    template <typename T>
    std::size_t operator()(T const &geom) const {
        return mapnik::util::apply_visitor(*this, geom);
    }

    std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
        return 0;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::point<T> const &) const {
        return 1;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
        return 1;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
        return 1;
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
        return mp.size();
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
        return mls.size();
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
        return mp.size();
    }

    template <typename T>
    std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
        std::size_t sum = 0;
        for (auto const &geom : col) {
            sum += operator()(geom);
        }
        return sum;
    }
};
} // namespace detail

template <typename T>
std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
    return detail::feature_count()(g);
}

void require_geometry(mapnik::feature_ptr feature,
                      std::size_t num_parts,
                      mapnik::geometry::geometry_types type) {
    REQUIRE(bool(feature));
    CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
    CHECK(feature_count(feature->get_geometry()) == num_parts);
}
} // anonymous namespace

static const std::string csv_plugin("./plugins/input/csv.input");

const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);

TEST_CASE("csv") {

    if (mapnik::util::exists(csv_plugin))
    {
        REQUIRE(registered);
        // make the tests silent since we intentially test error conditions that are noisy
        auto const severity = mapnik::logger::instance().get_severity();
        mapnik::logger::instance().set_severity(mapnik::logger::none);

        // check the CSV datasource is loaded
        const std::vector<std::string> plugin_names =
            mapnik::datasource_cache::instance().plugin_names();
        const bool have_csv_plugin =
            std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();

        SECTION("broken files") {
            if (have_csv_plugin) {
                std::vector<bfs::path> broken;
                add_csv_files("test/data/csv/fails", broken);
                add_csv_files("test/data/csv/warns", broken);
                broken.emplace_back("test/data/csv/fails/does_not_exist.csv");

                for (auto const &path : broken)
                {
                    INFO(path);
                    REQUIRE_THROWS(get_csv_ds(path.native()));
{'fraction_non_alphanumeric': 0.14780206386609246, 'fraction_numerical': 0.0014625822702527018, 'mean_word_length': 3.2662045060658578, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3369701', 'n_tokens_mistral': 4008, 'n_tokens_neox': 3909, 'n_words': 745}