<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2016 <NAME>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

#include "process_geojson_file_x3.hpp"

#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#include <mapnik/warning_ignore.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/streams/bufferstream.hpp>
#include <boost/spirit/home/x3.hpp>
#pragma GCC diagnostic pop
#include <mapnik/mapped_memory_cache.hpp>
#else
#include <mapnik/util/file_io.hpp>
#endif
#include <mapnik/feature.hpp>
#include <mapnik/json/json_grammar_config.hpp>
#include <mapnik/json/geojson_grammar_x3_def.hpp>
#include <mapnik/json/unicode_string_grammar_x3_def.hpp>
#include <mapnik/json/positions_grammar_x3_def.hpp>

namespace {

enum well_known_names
{
    type = 0,
    geometry = 1,
    coordinates = 2,
    properties = 3
};

constexpr int feature_properties[] = {
    well_known_names::type,
    well_known_names::geometry,
    well_known_names::properties}; // sorted

constexpr int geometry_properties[] = {
    well_known_names::type,
    well_known_names::coordinates}; // sorted

template <typename Keys>
std::string join(Keys const& keys)
{
    std::string result;
    bool first = true;
    for (auto const& key : keys)
    {
        if (!first) result += ",";
        result += "\"" + std::to_string(key) + "\"";
        first = false;
    }
    return result;
}

template <typename Iterator, typename Keys>
bool has_keys(Iterator first1, Iterator last1, Keys const& keys)
{
    auto first2 = std::begin(keys);
    auto last2 = std::end(keys);
    for (; first2 != last2; ++first1)
    {
        if (first1 == last1 || *first2 < std::get<0>(*first1))
            return false;
        if (!(std::get<0>(*first1) < *first2))
            ++first2;
    }
    return true;
}

template <typename Keys>
bool validate_geojson_feature(mapnik::json::geojson_value & value, Keys const& keys, bool verbose)
{
    if (!value.is<mapnik::json::geojson_object>())
    {
        if (verbose) std::clog << "Expecting an GeoJSON object" << std::endl;
        return false;
    }
    mapnik::json::geojson_object & feature = mapnik::util::get<mapnik::json::geojson_object>(value);
    std::sort(feature.begin(), feature.end(), [](auto const& e0, auto const& e1)
              {
                  return std::get<0>(e0) < std::get<0>(e1);
              });

    if (!has_keys(feature.begin(), feature.end(), feature_properties))
    {
        if (verbose) std::clog << "Expecting one of " << join(feature_properties) << std::endl;
        return false;
    }

    for (auto & elem : feature)
    {
        auto const key = std::get<0>(elem);
        if (key == well_known_names::geometry)
        {
            auto & geom_value = std::get<1>(elem);
            if (!geom_value.is<mapnik::json::geojson_object>())
            {
                if (verbose) std::clog << "\"geometry\": xxx <-- expecting an JSON object here" << std::endl;
                return false;
            }
            auto & geometry = mapnik::util::get<mapnik::json::geojson_object>(geom_value);
            // sort by property name
            std::sort(geometry.begin(), geometry.end(), [](auto const& e0, auto const& e1)
                      {
                          return std::get<0>(e0) < std::get<0>(e1);
                      });
            if (!has_keys(geometry.begin(), geometry.end(), geometry_properties))
            {
                if (verbose) std::clog << "Expecting one of " << join(geometry_properties) << std::endl;
                return false;
            }

            mapnik::geometry::geometry_types geom_type;
            mapnik::json::positions const* coordinates = nullptr;
            for (auto & elem2 : geometry)
            {
                auto const key2 = std::get<0>(elem2);
                if (key2 == well_known_names::type)
                {
                    auto const& geom_type_value = std::get<1>(elem2);
                    if (!geom_type_value.is<mapnik::geometry::geometry_types>())
                    {
                        if (verbose) std::clog << "\"type\": xxx <-- expecting an GeoJSON geometry type here" << std::endl;
                        return false;
                    }
                    geom_type = mapnik::util::get<mapnik::geometry::geometry_types>(geom_type_value);
                    if (geom_type == mapnik::geometry::geometry_types::GeometryCollection)
                    {
                        if (verbose) std::clog << "GeometryCollections are not allowed" << std::endl;;
                        return false;
                    }
                }
                else if (key2 == well_known_names::coordinates)
                {
                    auto const& coordinates_value = std::get<1>(elem2);
                    if (!coordinates_value.is<mapnik::json::positions>())
                    {
                        if (verbose) std::clog << "\"coordinates\": xxx <-- expecting an GeoJSON positions here" << std::endl;
                        return false;
                    }
                    coordinates = &mapnik::util::get<mapnik::json::positions>(coordinates_value);
                }
            }
            if (geom_type == mapnik::geometry::geometry_types::Point)
            {
                // expecting single position
                if (!coordinates->is<mapnik::json::point>())
                {
                    if (verbose) std::clog << "Expecting single position in Point" << std::endl;
                    return false;
                }
            }<commit_msg>add well_known_names to string mappings<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2016 <NAME>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/

#include "process_geojson_file_x3.hpp"

#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#include <mapnik/warning_ignore.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/streams/bufferstream.hpp>
#include <boost/spirit/home/x3.hpp>
#pragma GCC diagnostic pop
#include <mapnik/mapped_memory_cache.hpp>
#else
#include <mapnik/util/file_io.hpp>
#endif
#include <mapnik/feature.hpp>
#include <mapnik/json/json_grammar_config.hpp>
#include <mapnik/json/geojson_grammar_x3_def.hpp>
#include <mapnik/json/unicode_string_grammar_x3_def.hpp>
#include <mapnik/json/positions_grammar_x3_def.hpp>

namespace mapnik { namespace json {

constexpr char const* wkn_to_string(well_known_names val)
{
    switch(val)
    {
    case type: return "type";
    case geometry: return "geometry";
    case coordinates: return "coordinates";
    case properties: return "properties";
    default: return "unknown";
    }
}

}}

namespace {

constexpr mapnik::json::well_known_names feature_properties[] = {
    mapnik::json::well_known_names::type,
    mapnik::json::well_known_names::geometry,
    mapnik::json::well_known_names::properties}; // sorted

constexpr mapnik::json::well_known_names geometry_properties[] = {
    mapnik::json::well_known_names::type,
    mapnik::json::well_known_names::coordinates}; // sorted

template <typename Keys>
std::string join(Keys const& keys)
{
    std::string result;
    bool first = true;
    for (auto const& key : keys)
    {
        if (!first) result += ",";
        result += "\"" + std::string(mapnik::json::wkn_to_string(key)) + "\"";
        first = false;
    }
    return result;
}

template <typename Iterator, typename Keys>
bool has_keys(Iterator first1, Iterator last1, Keys const& keys)
{
    auto first2 = std::begin(keys);
    auto last2 = std::end(keys);
    for (; first2 != last2; ++first1)
    {
        if (first1 == last1 || *first2 < std::get<0>(*first1))
            return false;
        if (!(std::get<0>(*first1) < *first2))
            ++first2;
    }
    return true;
}

template <typename Keys>
bool validate_geojson_feature(mapnik::json::geojson_value & value, Keys const& keys, bool verbose)
{
    if (!value.is<mapnik::json::geojson_object>())
    {
        if (verbose) std::clog << "Expecting an GeoJSON object" << std::endl;
        return false;
    }
    mapnik::json::geojson_object & feature = mapnik::util::get<mapnik::json::geojson_object>(value);
    std::sort(feature.begin(), feature.end(), [](auto const& e0, auto const& e1)
              {
                  return std::get<0>(e0) < std::get<0>(e1);
              });

    if (!has_keys(feature.begin(), feature.end(), feature_properties))
    {
        if (verbose) std::clog << "Expecting one of " << join(feature_properties) << std::endl;
        return false;
    }

    for (auto & elem : feature)
    {
        auto const key = std::get<0>(elem);
        if (key == mapnik::json::well_known_names::geometry)
        {
            auto & geom_value = std::get<1>(elem);
            if (!geom_value.is<mapnik::json::geojson_object>())
            {
                if (verbose) std::clog << "\"geometry\": xxx <-- expecting an JSON object here" << std::endl;
                return false;
            }
            auto & geometry = mapnik::util::get<mapnik::json::geojson_object>(geom_value);
            // sort by property name
            std::sort(geometry.begin(), geometry.end(), [](auto const& e0, auto const& e1)
                      {
                          return std::get<0>(e0) < std::get<0>(e1);
                      });
            if (!has_keys(geometry.begin(), geometry.end(), geometry_properties))
            {
                if (verbose) std::clog << "Expecting one of " << join(geometry_properties) << std::endl;
                return false;
            }

            mapnik::geometry::geometry_types geom_type;
            mapnik::json::positions const* coordinates = nullptr;
            for (auto & elem2 : geometry)
            {
                auto const key2 = std::get<0>(elem2);
                if (key2 == mapnik::json::well_known_names::type)
                {
                    auto const& geom_type_value = std::get<1>(elem2);
                    if (!geom_type_value.is<mapnik::geometry::geometry_types>())
                    {
                        if (verbose) std::clog << "\"type\": xxx <-- expecting an GeoJSON geometry type here" << std::endl;
                        return false;
                    }
                    geom_type = mapnik::util::get<mapnik::geometry::geometry_types>(geom_type_value);
                    if (geom_type == mapnik::geometry::geometry_types::GeometryCollection)
                    {
                        if (verbose) std::clog << "GeometryCollections are not allowed" << std::endl;;
                        return false;
                    }
                }
                else if (key2 == mapnik::json::well_known_names::coordinates)
                {
                    auto const& coordinates_value = std::get<1>(elem2);
                    if (!coordinates_value.is<mapnik::json::positions>())
                    {
                        if (verbose) std::clog << "\"coordinates\": xxx <-- expecting an GeoJSON positions here" << std::endl;
                        return false;
                    }
                    coordinates = &mapnik::util::get<mapnik::json::positions>(coordinates_value);
                }
            }
            if (geom_type == mapnik::geometry::geometry_types::Point)
            {
                // expecting single position
                if (!coordinates->is<mapnik::json::point>())
                {
                    if (verbose) std::clog << "Expecting single position in Point" << std::endl;
                    return false;
                }
            }
{'fraction_non_alphanumeric': 0.14667064350160317, 'fraction_numerical': 0.0095444038475878, 'mean_word_length': 2.365621079046424, 'pattern_counts': {'":': 6, '<': 159, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7142202', 'n_tokens_mistral': 3881, 'n_tokens_neox': 3699, 'n_words': 943}