<commit_before>#include <vector>
#include <string>
#include <algorithm>
#include <sstream>

#include <boost/unordered_map.hpp>
#include <boost/assign/list_of.hpp>

#include <boost/python.hpp>
#include <boost/python/stl_iterator.hpp>

// We intentionally omit numpy/arrayobject.h here because cpp files need to be careful with exactly when this is imported.
// http://docs.scipy.org/doc/numpy/reference/c-api.array.html#importing-the-api
// Therefore, we assume the cpp file that included converters.hpp has already included numpy/arrayobject.h
// #include <numpy/arrayobject.h>

#include "BinaryData.h"
#include "DVIDException.h"

using namespace boost::python;

namespace libdvid { namespace python {

    //*********************************************************************************************
    //* Helpers
    //*********************************************************************************************

    //! PyBinaryDataHolder is the Python wrapper class for the SmartPtrHolder<BinaryDataPtr> class.
    //! It is exposed here as a global so helper functions can use it to manage BinaryData objects.
    //! It must be assigned in the module init section (see libdvid_python.cpp).
    boost::python::object PyBinaryDataHolder;
    
    template<class SmartPtr>
    struct SmartPtrHolder
    {
        SmartPtrHolder() {}
        SmartPtr ptr;
    };
    typedef SmartPtrHolder<libdvid::BinaryDataPtr> BinaryDataHolder;


    //*********************************************************************************************
    //*********************************************************************************************
    //* Python -> C++
    //*********************************************************************************************
    //*********************************************************************************************

    //!*********************************************************************************************
    //! This struct tells boost::python how to convert Python sequences into std::vector<T>.
    //! To use it, instantiate a single instance of it somewhere in your module init section:
    //!
    //!     libdvid::python::std_vector_from_python_iterable<unsigned int>();
    //!
    //! NOTE: This does NOT convert the other way, i.e. from std::vector<T> into Python list.
    //!      If you need to return a std::vector<T>, you'll have to implement that separately.
    //!
    //! For explanation and examples, see the following links:
    //! https://misspent.wordpress.com/2009/09/27/how-to-write-boost-python-converters
    //! http://www.boost.org/doc/libs/1_39_0/libs/python/doc/v2/faq.html#custom_string
    //!*********************************************************************************************
    template <typename T>
    struct std_vector_from_python_iterable
    {
        typedef std::vector<T> vector_t;
    
        std_vector_from_python_iterable()
        {
          converter::registry::push_back(
              &convertible,
              &construct,
              type_id<vector_t>());
        }
    
        // Determine if obj_ptr can be converted in a vector
        static void* convertible(PyObject* obj_ptr)
        {
            if (!PySequence_Check(obj_ptr))
            {
                return 0;
            }
            return obj_ptr;
        }
    
        // Convert obj_ptr into a vector
        static void construct( PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
        {
            assert( PySequence_Check(obj_ptr) );
    
            // Grab pointer to memory into which to construct the new vector_t
            void* storage = ((converter::rvalue_from_python_storage<vector_t>*) data)->storage.bytes;
    
            // in-place construct the new vector_t
            vector_t * the_vector = new (storage) vector_t;
    
            // Now copy the data from python into the vector
            typedef stl_input_iterator<T> vector_iter_t;
            object obj = object(handle<>(borrowed(obj_ptr)));
            the_vector->assign( vector_iter_t(obj), vector_iter_t()  );
    
            // Stash the memory chunk pointer for later use by boost::python
            data->convertible = storage;
        }
    };

    //!*********************************************************************************************
    //! This converter auto-converts 'None' objects into empty std::strings.
    //!*********************************************************************************************
    struct std_string_from_python_none
    {
        std_string_from_python_none()
        {
          converter::registry::push_back(
              &convertible,
              &construct,
              type_id<std::string>());
        }
    
        // Determine if obj_ptr is None
        static void* convertible(PyObject* obj_ptr)
        {
            if (obj_ptr == Py_None)
            {
                return obj_ptr;
            }
            return 0;
        }
    
        // Convert obj_ptr into a std::string
        static void construct( PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
        {
            assert (obj_ptr == Py_None);
    
            // Grab pointer to memory into which to construct the std::string
            void* storage = ((converter::rvalue_from_python_storage<std::string>*) data)->storage.bytes;
    
            // in-place construct the new std::string
            // extraced from the python object
            new (storage) std::string;
        
            // Stash the memory chunk pointer for later use by boost.python
            data->convertible = storage;
        }
    };

    //!*********************************************************************************************
    //! This helper struct is specialized over integer types to provide a
    //! compile-time mapping from integer types to numpy typenumbers.
    //!*********************************************************************************************
    template <typename T> struct numpy_typenums {};
    template <> struct numpy_typenums<libdvid::uint8> { static const int typenum = NPY_UINT8; };
    template <> struct numpy_typenums<libdvid::uint64> { static const int typenum = NPY_UINT64; };

    //!*********************************************************************************************
    //! Declares a mapping between numpy typenumbers and the corresponding dtype names
    //!*********************************************************************************************
    static boost::unordered_map<int, std::string> dtype_names =
        boost::assign::map_list_of
        (NPY_UINT8, "uint8")
        (NPY_UINT64, "uint64");

    //!*********************************************************************************************
    //! Converts the given numpy ndarray object into a DVIDVoxels object.
    //! NOTE: The data from the ndarray is *copied* into the new DVIDVoxels object.
    //!*********************************************************************************************
    template <class VolumeType>
    struct ndarray_to_volume
    {
        ndarray_to_volume()
        {
          converter::registry::push_back( &convertible, &construct, type_id<VolumeType>() );
          converter::registry::push_back( &convertible, &construct, type_id<VolumeType&>() );
        }
    
        static void* convertible(PyObject* obj_ptr)
        {
            if (!PyArray_Check(obj_ptr))
            {
                return 0;
            }
            // We could also check shape, dtype, etc. here, but it's more helpful to 
            // wait until construction time, and then throw a specific exception to
            // tell the user what he did wrong.
            return obj_ptr;
        }

        static void construct( PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
        {
            using namespace boost::python;
            using namespace libdvid;

            assert(PyArray_Check(obj_ptr));
    
            typedef typename VolumeType::voxel_type voxel_type;
    
            object ndarray = object(handle<>(borrowed(obj_ptr)));
            
            // Verify ndarray.dtype
            std::string dtype = extract<std::string>(str(ndarray.attr("dtype")));<commit_msg>python: Removed redundant converter registration.<commit_after>#include <vector>
#include <string>
#include <algorithm>
#include <sstream>

#include <boost/unordered_map.hpp>
#include <boost/assign/list_of.hpp>

#include <boost/python.hpp>
#include <boost/python/stl_iterator.hpp>

// We intentionally omit numpy/arrayobject.h here because cpp files need to be careful with exactly when this is imported.
// http://docs.scipy.org/doc/numpy/reference/c-api.array.html#importing-the-api
// Therefore, we assume the cpp file that included converters.hpp has already included numpy/arrayobject.h
// #include <numpy/arrayobject.h>

#include "BinaryData.h"
#include "DVIDException.h"

using namespace boost::python;

namespace libdvid { namespace python {

    //*********************************************************************************************
    //* Helpers
    //*********************************************************************************************

    //! PyBinaryDataHolder is the Python wrapper class for the SmartPtrHolder<BinaryDataPtr> class.
    //! It is exposed here as a global so helper functions can use it to manage BinaryData objects.
    //! It must be assigned in the module init section (see libdvid_python.cpp).
    boost::python::object PyBinaryDataHolder;
    
    template<class SmartPtr>
    struct SmartPtrHolder
    {
        SmartPtrHolder() {}
        SmartPtr ptr;
    };
    typedef SmartPtrHolder<libdvid::BinaryDataPtr> BinaryDataHolder;


    //*********************************************************************************************
    //*********************************************************************************************
    //* Python -> C++
    //*********************************************************************************************
    //*********************************************************************************************

    //!*********************************************************************************************
    //! This struct tells boost::python how to convert Python sequences into std::vector<T>.
    //! To use it, instantiate a single instance of it somewhere in your module init section:
    //!
    //!     libdvid::python::std_vector_from_python_iterable<unsigned int>();
    //!
    //! NOTE: This does NOT convert the other way, i.e. from std::vector<T> into Python list.
    //!      If you need to return a std::vector<T>, you'll have to implement that separately.
    //!
    //! For explanation and examples, see the following links:
    //! https://misspent.wordpress.com/2009/09/27/how-to-write-boost-python-converters
    //! http://www.boost.org/doc/libs/1_39_0/libs/python/doc/v2/faq.html#custom_string
    //!*********************************************************************************************
    template <typename T>
    struct std_vector_from_python_iterable
    {
        typedef std::vector<T> vector_t;
    
        std_vector_from_python_iterable()
        {
          converter::registry::push_back(
              &convertible,
              &construct,
              type_id<vector_t>());
        }
    
        // Determine if obj_ptr can be converted in a vector
        static void* convertible(PyObject* obj_ptr)
        {
            if (!PySequence_Check(obj_ptr))
            {
                return 0;
            }
            return obj_ptr;
        }
    
        // Convert obj_ptr into a vector
        static void construct( PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
        {
            assert( PySequence_Check(obj_ptr) );
    
            // Grab pointer to memory into which to construct the new vector_t
            void* storage = ((converter::rvalue_from_python_storage<vector_t>*) data)->storage.bytes;
    
            // in-place construct the new vector_t
            vector_t * the_vector = new (storage) vector_t;
    
            // Now copy the data from python into the vector
            typedef stl_input_iterator<T> vector_iter_t;
            object obj = object(handle<>(borrowed(obj_ptr)));
            the_vector->assign( vector_iter_t(obj), vector_iter_t()  );
    
            // Stash the memory chunk pointer for later use by boost::python
            data->convertible = storage;
        }
    };

    //!*********************************************************************************************
    //! This converter auto-converts 'None' objects into empty std::strings.
    //!*********************************************************************************************
    struct std_string_from_python_none
    {
        std_string_from_python_none()
        {
          converter::registry::push_back(
              &convertible,
              &construct,
              type_id<std::string>());
        }
    
        // Determine if obj_ptr is None
        static void* convertible(PyObject* obj_ptr)
        {
            if (obj_ptr == Py_None)
            {
                return obj_ptr;
            }
            return 0;
        }
    
        // Convert obj_ptr into a std::string
        static void construct( PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
        {
            assert (obj_ptr == Py_None);
    
            // Grab pointer to memory into which to construct the std::string
            void* storage = ((converter::rvalue_from_python_storage<std::string>*) data)->storage.bytes;
    
            // in-place construct the new std::string
            // extraced from the python object
            new (storage) std::string;
        
            // Stash the memory chunk pointer for later use by boost.python
            data->convertible = storage;
        }
    };

    //!*********************************************************************************************
    //! This helper struct is specialized over integer types to provide a
    //! compile-time mapping from integer types to numpy typenumbers.
    //!*********************************************************************************************
    template <typename T> struct numpy_typenums {};
    template <> struct numpy_typenums<libdvid::uint8> { static const int typenum = NPY_UINT8; };
    template <> struct numpy_typenums<libdvid::uint64> { static const int typenum = NPY_UINT64; };

    //!*********************************************************************************************
    //! Declares a mapping between numpy typenumbers and the corresponding dtype names
    //!*********************************************************************************************
    static boost::unordered_map<int, std::string> dtype_names =
        boost::assign::map_list_of
        (NPY_UINT8, "uint8")
        (NPY_UINT64, "uint64");

    //!*********************************************************************************************
    //! Converts the given numpy ndarray object into a DVIDVoxels object.
    //! NOTE: The data from the ndarray is *copied* into the new DVIDVoxels object.
    //!*********************************************************************************************
    template <class VolumeType>
    struct ndarray_to_volume
    {
        ndarray_to_volume()
        {
          converter::registry::push_back( &convertible, &construct, type_id<VolumeType>() );
        }
    
        static void* convertible(PyObject* obj_ptr)
        {
            if (!PyArray_Check(obj_ptr))
            {
                return 0;
            }
            // We could also check shape, dtype, etc. here, but it's more helpful to 
            // wait until construction time, and then throw a specific exception to
            // tell the user what he did wrong.
            return obj_ptr;
        }

        static void construct( PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data)
        {
            using namespace boost::python;
            using namespace libdvid;

            assert(PyArray_Check(obj_ptr));
    
            typedef typename VolumeType::voxel_type voxel_type;
    
            object ndarray = object(handle<>(borrowed(obj_ptr)));
            
            // Verify ndarray.dtype
            std::string dtype = extract<std::string>(str(ndarray.attr("dtype")));
{'fraction_non_alphanumeric': 0.26098471168893705, 'fraction_numerical': 0.0037317924641868303, 'mean_word_length': 2.855883035507078, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 84, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6242354', 'n_tokens_mistral': 4081, 'n_tokens_neox': 3875, 'n_words': 1057}