From 98a3dcf59cfa6746a31c2830ede979fd9f4df0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Thu, 1 Apr 2021 17:46:46 +0200 Subject: [PATCH] python: basic support for arbitrary types in strided array views. Not ArrayView yet, and also no documentation on this whatsoever. That'll come next. Also not everything works with arbitrary types yet, converting from buffer protocol doesn't remember the format and conversion to bytes doesn't take the actual type size into account either. --- src/Corrade/Containers/CMakeLists.txt | 8 +- .../StridedArrayViewPythonBindings.h | 159 ++++++++++++++++++ src/python/corrade/CMakeLists.txt | 4 + src/python/corrade/containers.cpp | 140 +++++++-------- src/python/corrade/test/CMakeLists.txt | 31 ++++ src/python/corrade/test/test_containers.py | 48 ++++++ .../corrade/test/test_containers_numpy.py | 132 +++++++++++++++ .../corrade/test/test_stridedarrayview.cpp | 89 ++++++++++ src/python/magnum/magnum.cpp | 5 +- src/python/magnum/trade.cpp | 3 +- 10 files changed, 544 insertions(+), 75 deletions(-) create mode 100644 src/Corrade/Containers/StridedArrayViewPythonBindings.h create mode 100644 src/python/corrade/test/CMakeLists.txt create mode 100644 src/python/corrade/test/test_containers_numpy.py create mode 100644 src/python/corrade/test/test_stridedarrayview.cpp diff --git a/src/Corrade/Containers/CMakeLists.txt b/src/Corrade/Containers/CMakeLists.txt index 2ac7a9b..14c52ab 100644 --- a/src/Corrade/Containers/CMakeLists.txt +++ b/src/Corrade/Containers/CMakeLists.txt @@ -24,7 +24,11 @@ # if(WITH_PYTHON) - add_custom_target(CorradeContainersPython SOURCES PythonBindings.h) + set(CorradeContainersPython_HEADERS + PythonBindings.h + StridedArrayViewPythonBindings.h) + + add_custom_target(CorradeContainersPython SOURCES ${CorradeContainersPython_HEADERS}) set_target_properties(CorradeContainersPython PROPERTIES FOLDER "Corrade/Python") - install(FILES PythonBindings.h DESTINATION ${CORRADE_INCLUDE_INSTALL_DIR}/Containers) + install(FILES ${CorradeContainersPython_HEADERS} DESTINATION ${CORRADE_INCLUDE_INSTALL_DIR}/Containers) endif() diff --git a/src/Corrade/Containers/StridedArrayViewPythonBindings.h b/src/Corrade/Containers/StridedArrayViewPythonBindings.h new file mode 100644 index 0000000..44ecaed --- /dev/null +++ b/src/Corrade/Containers/StridedArrayViewPythonBindings.h @@ -0,0 +1,159 @@ +#ifndef Corrade_Containers_StridedArrayViewPythonBindings_h +#define Corrade_Containers_StridedArrayViewPythonBindings_h +/* + This file is part of Magnum. + + Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, + 2020, 2021 Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include +#include + +namespace Corrade { namespace Containers { + +namespace Implementation { + +/* For maintainability please keep in the same order as + https://docs.python.org/3/library/struct.html#format-characters */ +template constexpr const char* formatString() { + static_assert(sizeof(T) == 0, "format string unknown for this type, supply it explicitly"); + return {}; +} +/* Representing bytes as unsigned. Not using 'c' because then it behaves + differently from bytes/bytearray, where you can do `a[0] = ord('A')`. */ +template<> constexpr const char* formatString() { return "B"; } +template<> constexpr const char* formatString() { return "b"; } +template<> constexpr const char* formatString() { return "B"; } +template<> constexpr const char* formatString() { return "h"; } +template<> constexpr const char* formatString() { return "H"; } +template<> constexpr const char* formatString() { return "i"; } +template<> constexpr const char* formatString() { return "I"; } +/* *not* l / L, that's 4 bytes in Python */ +template<> constexpr const char* formatString() { return "q"; } +template<> constexpr const char* formatString() { return "Q"; } +/** @todo how to represent std::size_t? conflicts with uint32_t/uint64_t above */ +/** @todo half? take from Magnum? */ +template<> constexpr const char* formatString() { return "f"; } +template<> constexpr const char* formatString() { return "d"; } + +template struct PyStridedArrayViewSetItem; +template struct PyStridedArrayViewSetItem { + /* __setitem__ is not even exposed for immutable views so this is fine */ + constexpr static std::nullptr_t set = nullptr; +}; +template struct PyStridedArrayViewSetItem { + static void set(char* item, pybind11::handle object) { + *reinterpret_cast(item) = pybind11::cast(object); + } +}; + +template struct PyStridedElement; + +} + +template class PyStridedArrayView: public StridedArrayView { + public: + /* Null function pointers should be okay as it shouldn't ever get to + them -- IndexError gets fired first. Not really sure about the + format, choosing bytes for safety. */ + /*implicit*/ PyStridedArrayView(): format{"B"}, getitem{} {} + + template explicit PyStridedArrayView(const StridedArrayView& view): PyStridedArrayView{view, Implementation::formatString::type>(), sizeof(U)} {} + + template explicit PyStridedArrayView(const StridedArrayView& view, const char* format, std::size_t itemsize): PyStridedArrayView{ + arrayCast(view), + format, + itemsize, + [](const char* item) { + return pybind11::cast(*reinterpret_cast(item)); + }, + Implementation::PyStridedArrayViewSetItem::set + } {} + + explicit PyStridedArrayView(const StridedArrayView& view, const char* format, std::size_t itemsize, pybind11::object(*getitem)(const char*), void(*setitem)(char*, pybind11::handle)): StridedArrayView{view}, format{format}, itemsize{itemsize}, getitem{getitem}, setitem{setitem} {} + + /* All APIs that are exposed by bindings and return a StridedArrayView + have to return the wrapper now */ + + typedef typename std::conditional>::type ElementType; + + ElementType operator[](std::size_t i) const { + return Implementation::PyStridedElement::wrap(StridedArrayView::operator[](i), format, itemsize, getitem, setitem); + } + + PyStridedArrayView slice(std::size_t begin, std::size_t end) const { + return PyStridedArrayView{StridedArrayView::slice(begin, end), format, itemsize, getitem, setitem}; + } + PyStridedArrayView slice(const typename StridedArrayView::Size& begin, const typename StridedArrayView::Size& end) const { + return PyStridedArrayView{StridedArrayView::slice(begin, end), format, itemsize, getitem, setitem}; + } + + /* slice() with templated dimensions not used */ + /* slice(&T::member) not used */ + /* prefix(), suffix(), except() not used */ + + PyStridedArrayView every(std::size_t skip) const { + return PyStridedArrayView{StridedArrayView::every(skip), format, itemsize, getitem, setitem}; + } + + PyStridedArrayView every(const typename StridedArrayView::Stride& skip) const { + return PyStridedArrayView{StridedArrayView::every(skip), format, itemsize, getitem, setitem}; + } + + template PyStridedArrayView transposed() const { + return PyStridedArrayView{StridedArrayView::template transposed(), format, itemsize, getitem, setitem}; + } + + template PyStridedArrayView flipped() const { + return PyStridedArrayView{StridedArrayView::template flipped(), format, itemsize, getitem, setitem}; + } + + template PyStridedArrayView broadcasted(std::size_t size) const { + return PyStridedArrayView{StridedArrayView::template broadcasted(size), format, itemsize, getitem, setitem}; + } + + /* has to be public as it's accessed by the bindings directly */ + const char* format; + std::size_t itemsize; + pybind11::object(*getitem)(const char*); + void(*setitem)(char*, pybind11::handle); +}; + +namespace Implementation { + +template struct PyStridedElement { + static PyStridedArrayView wrap(const StridedArrayView& element, const char* format, std::size_t itemsize, pybind11::object(*getitem)(const char*), void(*setitem)(char*, pybind11::handle)) { + return PyStridedArrayView{element, format, itemsize, getitem, setitem}; + } +}; + +template struct PyStridedElement<1, T> { + static T& wrap(T& element, const char*, std::size_t, pybind11::object(*)(const char*), void(*)(char*, pybind11::handle)) { + return element; + } +}; + +} + +}} + +#endif diff --git a/src/python/corrade/CMakeLists.txt b/src/python/corrade/CMakeLists.txt index d025f1c..e9685ac 100644 --- a/src/python/corrade/CMakeLists.txt +++ b/src/python/corrade/CMakeLists.txt @@ -96,3 +96,7 @@ set_target_properties(corrade PROPERTIES file(GENERATE OUTPUT ${output_dir}/corrade/__init__.py INPUT ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py) + +if(BUILD_TESTS) + add_subdirectory(test) +endif() diff --git a/src/python/corrade/containers.cpp b/src/python/corrade/containers.cpp index 2112b3b..8fbe7e7 100644 --- a/src/python/corrade/containers.cpp +++ b/src/python/corrade/containers.cpp @@ -29,6 +29,7 @@ #include #include "Corrade/Containers/PythonBindings.h" +#include "Corrade/Containers/StridedArrayViewPythonBindings.h" #include "corrade/bootstrap.h" #include "corrade/PyBuffer.h" @@ -37,14 +38,6 @@ namespace corrade { namespace { -const char* const FormatStrings[]{ - /* 0. Representing bytes as unsigned. Not using 'c' because then it behaves - differently from bytes/bytearray, where you can do `a[0] = ord('A')`. */ - "B", -}; -template constexpr std::size_t formatIndex(); -template<> constexpr std::size_t formatIndex() { return 0; } - struct Slice { std::size_t start; std::size_t stop; @@ -84,7 +77,7 @@ template bool arrayViewBufferProtocol(T& self, Py_buffer& buffer, int f buffer.buf = const_cast::type*>(self.data()); buffer.readonly = std::is_const::value; if((flags & PyBUF_FORMAT) == PyBUF_FORMAT) - buffer.format = const_cast(FormatStrings[formatIndex::type>()]); + buffer.format = const_cast(Containers::Implementation::formatString::type>()); if(flags != PyBUF_SIMPLE) { /* The view is immutable (can't change its size after it has been constructed), so referencing the size directly is okay */ @@ -157,8 +150,10 @@ template void arrayView(py::class_, Containers const Slice calculated = calculateSlice(slice, self.size()); /* Non-trivial stride, return a different type */ + /** @todo this always assumes bytes for now -- remember the format + and provide a checked typed conversion API */ if(calculated.step != 1) { - auto sliced = Containers::stridedArrayView(self).slice(calculated.start, calculated.stop).every(calculated.step); + auto sliced = Containers::PyStridedArrayView<1, T>{ Containers::stridedArrayView(self)}.slice(calculated.start, calculated.stop).every(calculated.step); return pyCastButNotShitty(Containers::pyArrayViewHolder(sliced, sliced.size() ? pyObjectHolderFor(self).owner : py::none{})); } @@ -290,14 +285,14 @@ template bool stridedArrayViewBufferProtocol(T& self, Py_buffer& buffer /* I hate the const_casts but I assume this is to make editing easier, NOT to make it possible for users to stomp on these values. */ buffer.ndim = T::Dimensions; - buffer.itemsize = sizeof(typename T::Type); - buffer.len = sizeof(typename T::Type); + buffer.itemsize = self.itemsize; + buffer.len = self.itemsize; for(std::size_t i = 0; i != T::Dimensions; ++i) buffer.len *= Containers::Implementation::sizeRef(self)[i]; buffer.buf = const_cast::type*>(self.data()); buffer.readonly = std::is_const::value; if((flags & PyBUF_FORMAT) == PyBUF_FORMAT) - buffer.format = const_cast(FormatStrings[formatIndex::type>()]); + buffer.format = const_cast(self.format); /* The view is immutable (can't change its size after it has been constructed), so referencing the size/stride directly is okay */ buffer.shape = const_cast(reinterpret_cast(Containers::Implementation::sizeRef(self).begin())); @@ -310,11 +305,11 @@ inline std::size_t largerStride(std::size_t a, std::size_t b) { return a < b ? b : a; /* max(), but named like this to avoid clashes */ } -template void stridedArrayView(py::class_, Containers::PyArrayViewHolder>>& c) { +template void stridedArrayView(py::class_, Containers::PyArrayViewHolder>>& c) { /* Implicitly convertible from a buffer */ - py::implicitly_convertible>(); + py::implicitly_convertible>(); /* This is needed for implicit conversion from np.array */ - py::implicitly_convertible>(); + py::implicitly_convertible>(); c /* Constructor */ @@ -347,63 +342,68 @@ template void stridedArrayView(py::class_{ + /** @todo this always assumes bytes for now -- remember the format + and provide a checked typed conversion API */ + return Containers::pyArrayViewHolder(Containers::PyStridedArrayView{Containers::StridedArrayView{ {static_cast(buffer.buf), size}, Containers::StaticArrayView{reinterpret_cast(buffer.shape)}, - Containers::StaticArrayView{reinterpret_cast(buffer.strides)}}, + Containers::StaticArrayView{reinterpret_cast(buffer.strides)}}}, buffer.len ? py::reinterpret_borrow(buffer.obj) : py::none{}); }), "Construct from a buffer") /* Length, size/stride tuple, dimension count and memory owning object */ - .def("__len__", [](const Containers::StridedArrayView& self) { + .def("__len__", [](const Containers::PyStridedArrayView& self) { return Containers::StridedDimensions(self.size())[0]; }, "View size in the top-level dimension") - .def_property_readonly("size", [](const Containers::StridedArrayView& self) { + .def_property_readonly("size", [](const Containers::PyStridedArrayView& self) { return size(self.size()); }, "View size in each dimension") - .def_property_readonly("stride", [](const Containers::StridedArrayView& self) { + .def_property_readonly("stride", [](const Containers::PyStridedArrayView& self) { return stride(self.stride()); }, "View stride in each dimension") - .def_property_readonly("dimensions", [](const Containers::StridedArrayView&) { return dimensions; }, "Dimension count") - .def_property_readonly("owner", [](const Containers::StridedArrayView& self) { + .def_property_readonly("dimensions", [](const Containers::PyStridedArrayView&) { return dimensions; }, "Dimension count") + .def_property_readonly("format", [](const Containers::PyStridedArrayView& self) { + return self.format; + }, "Format of each item") + .def_property_readonly("owner", [](const Containers::PyStridedArrayView& self) { return pyObjectHolderFor(self).owner; }, "Memory owner object") /* Conversion to bytes */ - .def("__bytes__", [](const Containers::StridedArrayView& self) { + .def("__bytes__", [](const Containers::PyStridedArrayView& self) { /* TODO: use _PyBytes_Resize() to avoid the double copy */ const Containers::Array out = bytes(Containers::arrayCast(self)); return py::bytes(out.data(), out.size()); }, "Convert to bytes") /* Slicing of the top dimension */ - .def("__getitem__", [](const Containers::StridedArrayView& self, py::slice slice) { + .def("__getitem__", [](const Containers::PyStridedArrayView& self, py::slice slice) { const Slice calculated = calculateSlice(slice, Containers::StridedDimensions{self.size()}[0]); const auto sliced = self.slice(calculated.start, calculated.stop).every(calculated.step); return Containers::pyArrayViewHolder(sliced, calculated.start == calculated.stop ? py::none{} : pyObjectHolderFor(self).owner); }, "Slice the view"); - enableBetterBufferProtocol, stridedArrayViewBufferProtocol>(c); + enableBetterBufferProtocol, stridedArrayViewBufferProtocol>(c); } -template void stridedArrayView1D(py::class_, Containers::PyArrayViewHolder>>& c) { +template void stridedArrayView1D(py::class_, Containers::PyArrayViewHolder>>& c) { c /* Single item retrieval. Need to raise IndexError in order to allow iteration: https://docs.python.org/3/reference/datamodel.html#object.__getitem__ */ - .def("__getitem__", [](const Containers::StridedArrayView<1, T>& self, std::size_t i) { + .def("__getitem__", [](const Containers::PyStridedArrayView<1, T>& self, std::size_t i) { if(i >= self.size()) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; } - return self[i]; + return self.getitem(&self[i]); }, "Value at given position"); } -template void stridedArrayViewND(py::class_, Containers::PyArrayViewHolder>>& c) { +template void stridedArrayViewND(py::class_, Containers::PyArrayViewHolder>>& c) { c /* Sub-view retrieval. Need to raise IndexError in order to allow iteration: https://docs.python.org/3/reference/datamodel.html#object.__getitem__ */ - .def("__getitem__", [](const Containers::StridedArrayView& self, std::size_t i) { + .def("__getitem__", [](const Containers::PyStridedArrayView& self, std::size_t i) { if(i >= Containers::StridedDimensions{self.size()}[0]) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; @@ -412,7 +412,7 @@ template void stridedArrayViewND(py::class_& self, const typename DimensionsTuple::Type& slice) { + .def("__getitem__", [](const Containers::PyStridedArrayView& self, const typename DimensionsTuple::Type& slice) { Containers::StridedDimensions starts; Containers::StridedDimensions stops; Containers::StridedDimensions steps; @@ -432,26 +432,26 @@ template void stridedArrayViewND(py::class_ void stridedArrayView2D(py::class_, Containers::PyArrayViewHolder>>& c) { +template void stridedArrayView2D(py::class_, Containers::PyArrayViewHolder>>& c) { c /* Single-item retrieval. Need to raise IndexError in order to allow iteration: https://docs.python.org/3/reference/datamodel.html#object.__getitem__ */ - .def("__getitem__", [](const Containers::StridedArrayView<2, T>& self, const std::tuple& i) { + .def("__getitem__", [](const Containers::PyStridedArrayView<2, T>& self, const std::tuple& i) { if(std::get<0>(i) >= self.size()[0] || std::get<1>(i) >= self.size()[1]) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; } - return self[std::get<0>(i)][std::get<1>(i)]; + return self.getitem(&self[std::get<0>(i)][std::get<1>(i)]); }, "Value at given position") - .def("transposed", [](const Containers::StridedArrayView<2, T>& self, const std::size_t a, std::size_t b) { + .def("transposed", [](const Containers::PyStridedArrayView<2, T>& self, const std::size_t a, std::size_t b) { if((a == 0 && b == 1) || (a == 1 && b == 0)) return Containers::pyArrayViewHolder(self.template transposed<0, 1>(), pyObjectHolderFor(self).owner); PyErr_Format(PyExc_ValueError, "dimensions %zu, %zu can't be transposed in a %iD view", a, b, 2); throw py::error_already_set{}; }, "Transpose two dimensions") - .def("flipped", [](const Containers::StridedArrayView<2, T>& self, const std::size_t dimension) { + .def("flipped", [](const Containers::PyStridedArrayView<2, T>& self, const std::size_t dimension) { if(dimension == 0) return Containers::pyArrayViewHolder(self.template flipped<0>(), pyObjectHolderFor(self).owner); if(dimension == 1) @@ -459,7 +459,7 @@ template void stridedArrayView2D(py::class_& self, const std::size_t dimension, std::size_t size) { + .def("broadcasted", [](const Containers::PyStridedArrayView<2, T>& self, const std::size_t dimension, std::size_t size) { if(dimension == 0) return Containers::pyArrayViewHolder(self.template broadcasted<0>(size), pyObjectHolderFor(self).owner); if(dimension == 1) @@ -469,20 +469,20 @@ template void stridedArrayView2D(py::class_ void stridedArrayView3D(py::class_, Containers::PyArrayViewHolder>>& c) { +template void stridedArrayView3D(py::class_, Containers::PyArrayViewHolder>>& c) { c /* Single-item retrieval. Need to raise IndexError in order to allow iteration: https://docs.python.org/3/reference/datamodel.html#object.__getitem__ */ - .def("__getitem__", [](const Containers::StridedArrayView<3, T>& self, const std::tuple& i) { + .def("__getitem__", [](const Containers::PyStridedArrayView<3, T>& self, const std::tuple& i) { if(std::get<0>(i) >= self.size()[0] || std::get<1>(i) >= self.size()[1] || std::get<2>(i) >= self.size()[2]) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; } - return self[std::get<0>(i)][std::get<1>(i)][std::get<2>(i)]; + return self.getitem(&self[std::get<0>(i)][std::get<1>(i)][std::get<2>(i)]); }, "Value at given position") - .def("transposed", [](const Containers::StridedArrayView<3, T>& self, const std::size_t a, std::size_t b) { + .def("transposed", [](const Containers::PyStridedArrayView<3, T>& self, const std::size_t a, std::size_t b) { if((a == 0 && b == 1) || (a == 1 && b == 0)) return Containers::pyArrayViewHolder(self.template transposed<0, 1>(), pyObjectHolderFor(self).owner); @@ -495,7 +495,7 @@ template void stridedArrayView3D(py::class_& self, const std::size_t dimension) { + .def("flipped", [](const Containers::PyStridedArrayView<3, T>& self, const std::size_t dimension) { if(dimension == 0) return Containers::pyArrayViewHolder(self.template flipped<0>(), pyObjectHolderFor(self).owner); if(dimension == 1) @@ -505,7 +505,7 @@ template void stridedArrayView3D(py::class_& self, const std::size_t dimension, std::size_t size) { + .def("broadcasted", [](const Containers::PyStridedArrayView<3, T>& self, const std::size_t dimension, std::size_t size) { if(dimension == 0) return Containers::pyArrayViewHolder(self.template broadcasted<0>(size), pyObjectHolderFor(self).owner); if(dimension == 1) @@ -517,11 +517,11 @@ template void stridedArrayView3D(py::class_ void stridedArrayView4D(py::class_, Containers::PyArrayViewHolder>>& c) { +template void stridedArrayView4D(py::class_, Containers::PyArrayViewHolder>>& c) { c /* Single-item retrieval. Need to raise IndexError in order to allow iteration: https://docs.python.org/3/reference/datamodel.html#object.__getitem__ */ - .def("__getitem__", [](const Containers::StridedArrayView<4, T>& self, const std::tuple& i) { + .def("__getitem__", [](const Containers::PyStridedArrayView<4, T>& self, const std::tuple& i) { if(std::get<0>(i) >= self.size()[0] || std::get<1>(i) >= self.size()[1] || std::get<2>(i) >= self.size()[2] || @@ -529,9 +529,9 @@ template void stridedArrayView4D(py::class_(i)][std::get<1>(i)][std::get<2>(i)][std::get<3>(i)]; + return self.getitem(&self[std::get<0>(i)][std::get<1>(i)][std::get<2>(i)][std::get<3>(i)]); }, "Value at given position") - .def("transposed", [](const Containers::StridedArrayView<4, T>& self, const std::size_t a, std::size_t b) { + .def("transposed", [](const Containers::PyStridedArrayView<4, T>& self, const std::size_t a, std::size_t b) { if((a == 0 && b == 1) || (a == 1 && b == 0)) return Containers::pyArrayViewHolder(self.template transposed<0, 1>(), pyObjectHolderFor(self).owner); @@ -553,7 +553,7 @@ template void stridedArrayView4D(py::class_& self, const std::size_t dimension) { + .def("flipped", [](const Containers::PyStridedArrayView<4, T>& self, const std::size_t dimension) { if(dimension == 0) return Containers::pyArrayViewHolder(self.template flipped<0>(), pyObjectHolderFor(self).owner); if(dimension == 1) @@ -565,7 +565,7 @@ template void stridedArrayView4D(py::class_& self, const std::size_t dimension, std::size_t size) { + .def("broadcasted", [](const Containers::PyStridedArrayView<4, T>& self, const std::size_t dimension, std::size_t size) { if(dimension == 0) return Containers::pyArrayViewHolder(self.template broadcasted<0>(size), pyObjectHolderFor(self).owner); if(dimension == 1) @@ -579,45 +579,45 @@ template void stridedArrayView4D(py::class_ void mutableStridedArrayView1D(py::class_, Containers::PyArrayViewHolder>>& c) { +void mutableStridedArrayView1D(py::class_, Containers::PyArrayViewHolder>>& c) { c - .def("__setitem__", [](const Containers::StridedArrayView<1, T>& self, const std::size_t i, const T& value) { + .def("__setitem__", [](const Containers::PyStridedArrayView<1, char>& self, const std::size_t i, py::handle value) { if(i >= self.size()) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; } - self[i] = value; + self.setitem(&self[i], value); }, "Set a value at given position"); } -template void mutableStridedArrayView2D(py::class_, Containers::PyArrayViewHolder>>& c) { +void mutableStridedArrayView2D(py::class_, Containers::PyArrayViewHolder>>& c) { c - .def("__setitem__", [](const Containers::StridedArrayView<2, T>& self, const std::tuple& i, const T& value) { + .def("__setitem__", [](const Containers::PyStridedArrayView<2, char>& self, const std::tuple& i, py::handle value) { if(std::get<0>(i) >= self.size()[0] || std::get<1>(i) >= self.size()[1]) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; } - self[std::get<0>(i)][std::get<1>(i)] = value; + self.setitem(&self[std::get<0>(i)][std::get<1>(i)], value); }, "Set a value at given position"); } -template void mutableStridedArrayView3D(py::class_, Containers::PyArrayViewHolder>>& c) { +void mutableStridedArrayView3D(py::class_, Containers::PyArrayViewHolder>>& c) { c - .def("__setitem__", [](const Containers::StridedArrayView<3, T>& self, const std::tuple& i, const T& value) { + .def("__setitem__", [](const Containers::PyStridedArrayView<3, char>& self, const std::tuple& i, py::handle value) { if(std::get<0>(i) >= self.size()[0] || std::get<1>(i) >= self.size()[1] || std::get<2>(i) >= self.size()[2]) { PyErr_SetNone(PyExc_IndexError); throw py::error_already_set{}; } - self[std::get<0>(i)][std::get<1>(i)][std::get<2>(i)] = value; + self.setitem(&self[std::get<0>(i)][std::get<1>(i)][std::get<2>(i)], value); }, "Set a value at given position"); } -template void mutableStridedArrayView4D(py::class_, Containers::PyArrayViewHolder>>& c) { +void mutableStridedArrayView4D(py::class_, Containers::PyArrayViewHolder>>& c) { c - .def("__setitem__", [](const Containers::StridedArrayView<4, T>& self, const std::tuple& i, const T& value) { + .def("__setitem__", [](const Containers::PyStridedArrayView<4, char>& self, const std::tuple& i, py::handle value) { if(std::get<0>(i) >= self.size()[0] || std::get<1>(i) >= self.size()[1] || std::get<2>(i) >= self.size()[2] || @@ -625,7 +625,7 @@ template void mutableStridedArrayView4D(py::class_(i)][std::get<1>(i)][std::get<2>(i)][std::get<3>(i)] = value; + self.setitem(&self[std::get<0>(i)][std::get<1>(i)][std::get<2>(i)][std::get<3>(i)], value); }, "Set a value at given position"); } @@ -643,13 +643,13 @@ void containers(py::module_& m) { arrayView(mutableArrayView_); mutableArrayView(mutableArrayView_); - py::class_, Containers::PyArrayViewHolder>> stridedArrayView1D_{m, + py::class_, Containers::PyArrayViewHolder>> stridedArrayView1D_{m, "StridedArrayView1D", "One-dimensional array view with stride information", py::buffer_protocol{}}; - py::class_, Containers::PyArrayViewHolder>> stridedArrayView2D_{m, + py::class_, Containers::PyArrayViewHolder>> stridedArrayView2D_{m, "StridedArrayView2D", "Two-dimensional array view with stride information", py::buffer_protocol{}}; - py::class_, Containers::PyArrayViewHolder>> stridedArrayView3D_{m, + py::class_, Containers::PyArrayViewHolder>> stridedArrayView3D_{m, "StridedArrayView3D", "Three-dimensional array view with stride information", py::buffer_protocol{}}; - py::class_, Containers::PyArrayViewHolder>> stridedArrayView4D_{m, + py::class_, Containers::PyArrayViewHolder>> stridedArrayView4D_{m, "StridedArrayView4D", "Four-dimensional array view with stride information", py::buffer_protocol{}}; stridedArrayView(stridedArrayView1D_); stridedArrayView1D(stridedArrayView1D_); @@ -663,13 +663,13 @@ void containers(py::module_& m) { stridedArrayViewND(stridedArrayView4D_); stridedArrayView4D(stridedArrayView4D_); - py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView1D_{m, + py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView1D_{m, "MutableStridedArrayView1D", "Mutable one-dimensional array view with stride information", py::buffer_protocol{}}; - py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView2D_{m, + py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView2D_{m, "MutableStridedArrayView2D", "Mutable two-dimensional array view with stride information", py::buffer_protocol{}}; - py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView3D_{m, + py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView3D_{m, "MutableStridedArrayView3D", "Mutable three-dimensional array view with stride information", py::buffer_protocol{}}; - py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView4D_{m, + py::class_, Containers::PyArrayViewHolder>> mutableStridedArrayView4D_{m, "MutableStridedArrayView4D", "Mutable four-dimensional array view with stride information", py::buffer_protocol{}}; stridedArrayView(mutableStridedArrayView1D_); stridedArrayView1D(mutableStridedArrayView1D_); diff --git a/src/python/corrade/test/CMakeLists.txt b/src/python/corrade/test/CMakeLists.txt new file mode 100644 index 0000000..9bcca69 --- /dev/null +++ b/src/python/corrade/test/CMakeLists.txt @@ -0,0 +1,31 @@ +# +# This file is part of Magnum. +# +# Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, +# 2020, 2021 Vladimír Vondruš +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# + +pybind11_add_module(test_stridedarrayview ${pybind11_add_module_SYSTEM} test_stridedarrayview.cpp) +target_include_directories(test_stridedarrayview PRIVATE ${PROJECT_SOURCE_DIR}/src) +target_link_libraries(test_stridedarrayview PRIVATE Corrade::Containers) +set_target_properties(test_stridedarrayview PROPERTIES + FOLDER "python" + LIBRARY_OUTPUT_DIRECTORY ${output_dir}) diff --git a/src/python/corrade/test/test_containers.py b/src/python/corrade/test/test_containers.py index 6f87fc0..62ecc6c 100644 --- a/src/python/corrade/test/test_containers.py +++ b/src/python/corrade/test/test_containers.py @@ -28,6 +28,7 @@ import sys import unittest from corrade import containers +import test_stridedarrayview class ArrayView(unittest.TestCase): def test_init(self): @@ -852,3 +853,50 @@ class StridedArrayView4D(unittest.TestCase): self.assertEqual(f.size, (2, 1, 3, 5)) self.assertEqual(f.stride, (24, 24, 8, 0)) self.assertEqual(bytes(f), b'000004444488888ccccc0000044444') + +class StridedArrayViewCustomType(unittest.TestCase): + def test_short(self): + a = test_stridedarrayview.get_containers() + self.assertEqual(type(a.view), containers.StridedArrayView2D) + self.assertEqual(a.view.size, (2, 3)) + self.assertEqual(a.view.stride, (3*2, 2)) + self.assertEqual(a.view.format, 'h') + self.assertEqual(a.list, [3, -17565, 5, 3, -17565, 5]) + self.assertEqual(a.view[0][0], 3) + self.assertEqual(a.view[0][1], -17565) + self.assertEqual(a.view[0][2], 5) + self.assertEqual(a.view[1][0], 3) + self.assertEqual(a.view[1][1], -17565) + self.assertEqual(a.view[1][2], 5) + + with self.assertRaisesRegex(TypeError, "object does not support item assignment"): + a.view[1][1] = 15 + + # Test that memoryview understands the type + av = memoryview(a.view[0]) + self.assertEqual(av[0], 3) + self.assertEqual(av[1], -17565) + self.assertEqual(av[2], 5) + + def test_mutable_int(self): + a = test_stridedarrayview.MutableContaineri() + self.assertEqual(type(a.view), containers.MutableStridedArrayView2D) + self.assertEqual(a.view.format, 'i') + self.assertEqual(a.list, [0, 0, 0, 0, 0, 0]) + a.view[0][1] = -7656581 + a.view[1][2] = 4666 + self.assertEqual(a.list, [0, -7656581, 0, 0, 0, 4666]) + + # Test that memoryview understands the type and has changes reflected + av = memoryview(a.view[1]) + a.view[1][0] = -333 + self.assertEqual(av[0], -333) + self.assertEqual(av[1], 0) + self.assertEqual(av[2], 4666) + + # And the other way around as well + av[1] = 11111 + self.assertEqual(a.list, [0, -7656581, 0, -333, 11111, 4666]) + + # mutable_vector3d and mutable_long_float tested in test_containers_numpy + # as memoryview can't handle their types diff --git a/src/python/corrade/test/test_containers_numpy.py b/src/python/corrade/test/test_containers_numpy.py new file mode 100644 index 0000000..0891d55 --- /dev/null +++ b/src/python/corrade/test/test_containers_numpy.py @@ -0,0 +1,132 @@ +# +# This file is part of Magnum. +# +# Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, +# 2020, 2021 Vladimír Vondruš +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# + +import unittest + +from corrade import containers +import test_stridedarrayview + +try: + import numpy as np +except ModuleNotFoundError: + raise unittest.SkipTest("numpy not installed") + +class StridedArrayViewCustomType(unittest.TestCase): + # short and mutable_int tested in test_containers, as for those memoryview + # works well... well, for one dimension it does + + def test_mutable_vector3d(self): + a = test_stridedarrayview.MutableContainer3d() + self.assertEqual(type(a.view), containers.MutableStridedArrayView2D) + self.assertEqual(a.view.format, 'ddd') + self.assertEqual(a.list, [ + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0] + ]) + a.view[0][1] = [-765.6581, 3.5, 1.125] + a.view[1][2] = [4.666, 0.25, -7.5] + self.assertEqual(a.list, [ + [0.0, 0.0, 0.0], + [-765.6581, 3.5, 1.125], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [4.666, 0.25, -7.5] + ]) + + # memoryview ... doesn't understand the type. HAH + mav = memoryview(a.view[0]) + with self.assertRaisesRegex(NotImplementedError, "unsupported format ddd"): + self.assertEqual(mav[1], [-765.6581, 3.5, 1.125]) + + # Test that numpy understands the type and has changes reflected + av = np.array(a.view, copy=False) + a.view[1][0] = [-3.33, 1.0, 0.0] + # Converting to a tuple, otherwise numpy always compares to False + self.assertEqual(tuple(av[1][0]), (-3.33, 1.0, 0.0)) + self.assertEqual(tuple(av[1][1]), (0.0, 0.0, 0.0)) + self.assertEqual(tuple(av[1][2]), (4.666, 0.25, -7.5)) + + # And the other way around as well + av[1][1] = (1.0, 0.125, 1.125) + self.assertEqual(a.list, [ + [0.0, 0.0, 0.0], + [-765.6581, 3.5, 1.125], + [0.0, 0.0, 0.0], + [-3.33, 1.0, 0.0], + [1.0, 0.125, 1.125], + [4.666, 0.25, -7.5] + ]) + + def test_mutable_long_float(self): + a = test_stridedarrayview.MutableContainerlf() + self.assertEqual(type(a.view), containers.MutableStridedArrayView2D) + self.assertEqual(a.view.format, 'Qf') + self.assertEqual(a.list, [ + (0, 0.0), + (0, 0.0), + (0, 0.0), + (0, 0.0), + (0, 0.0), + (0, 0.0) + ]) + a.view[0][1] = (7656581356781257, 1.125) + a.view[1][2] = (4666025, -7.5) + self.assertEqual(a.list, [ + (0, 0.0), + (7656581356781257, 1.125), + (0, 0.0), + (0, 0.0), + (0, 0.0), + (4666025, -7.5) + ]) + + # memoryview ... doesn't understand the type. HAH + mav = memoryview(a.view[0]) + with self.assertRaisesRegex(NotImplementedError, "unsupported format Qf"): + self.assertEqual(mav[1], (7656581356781257, 1.125)) + + # Test that numpy understands the type and has changes reflected + av = np.array(a.view, copy=False) + a.view[1][0] = (333106832, 0.0) + # Converting to a tuple, otherwise numpy always compares to False + self.assertEqual(tuple(av[1][0]), (333106832, 0.0)) + self.assertEqual(tuple(av[1][1]), (0, 0.0)) + self.assertEqual(tuple(av[1][2]), (4666025, -7.5)) + + # And the other way around as well + av[1][1] = (1001, 1.125) + self.assertEqual(a.list, [ + (0, 0.0), + (7656581356781257, 1.125), + (0, 0.0), + (333106832, 0.0), + (1001, 1.125), + (4666025, -7.5) + ]) diff --git a/src/python/corrade/test/test_stridedarrayview.cpp b/src/python/corrade/test/test_stridedarrayview.cpp new file mode 100644 index 0000000..d1e0e47 --- /dev/null +++ b/src/python/corrade/test/test_stridedarrayview.cpp @@ -0,0 +1,89 @@ +/* + This file is part of Magnum. + + Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, + 2020, 2021 Vladimír Vondruš + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +#include +#include + +#include "../bootstrap.h" /* for module / _module alias */ + +#include "Corrade/Containers/PythonBindings.h" +#include "Corrade/Containers/StridedArrayViewPythonBindings.h" + +namespace Corrade { namespace Containers { namespace Implementation { + template<> constexpr const char* formatString>() { + return "ddd"; + } + template<> constexpr const char* formatString>() { + return "Qf"; + } +}}} + +using namespace Corrade; +namespace py = pybind11; + +template struct Container { + Container(T a = {}, T b = {}, T c = {}): data{a, b, c, a, b, c} {} + + Containers::StridedArrayView2D view() { + return {Containers::arrayView(data), {2, 3}}; + } + + std::vector::type> list() const { + return {data, data + 6}; + } + + T data[3*2]{}; +}; + +template void container(py::class_>& c) { + c + .def(py::init()) + .def_property_readonly("view", [](Container& self) { + return Containers::pyArrayViewHolder(Containers::PyStridedArrayView<2, typename std::conditional::value, const char, char>::type>{self.view()}, py::cast(self)); + }) + .def_property_readonly("list", &Container::list); +} + +/* TODO: remove declaration when https://github.com/pybind/pybind11/pull/1863 + is released */ +extern "C" PYBIND11_EXPORT PyObject* PyInit_test_stridedarrayview(); +PYBIND11_MODULE(test_stridedarrayview, m) { + /* These are a part of the same module in the static build, no need to + import (also can't import because there it's _magnum.*) */ + py::module_::import("corrade.containers"); + + py::class_> containers{m, "Containers"}; + py::class_> mutableContaineri{m, "MutableContaineri"}; + py::class_>> mutableContainer3d{m, "MutableContainer3d"}; + py::class_>> mutableContainerlf{m, "MutableContainerlf"}; + container(containers); + container(mutableContaineri); + container(mutableContainer3d); + container(mutableContainerlf); + + m.def("get_containers", []() { + return Container{3, -17565, 5}; + }); +} diff --git a/src/python/magnum/magnum.cpp b/src/python/magnum/magnum.cpp index 1089212..d5da71b 100644 --- a/src/python/magnum/magnum.cpp +++ b/src/python/magnum/magnum.cpp @@ -35,6 +35,7 @@ #include "Corrade/PythonBindings.h" #include "Corrade/Containers/PythonBindings.h" +#include "Corrade/Containers/StridedArrayViewPythonBindings.h" #include "Magnum/PythonBindings.h" #include "magnum/bootstrap.h" @@ -66,7 +67,7 @@ template void image(py::class_& c) { return Containers::pyArrayViewHolder(self.data(), self.data() ? py::cast(self) : py::none{}); }, "Image data") .def_property_readonly("pixels", [](T& self) { - return Containers::pyArrayViewHolder(self.pixels(), self.data() ? py::cast(self) : py::none{}); + return Containers::pyArrayViewHolder(Containers::PyStridedArrayView{self.pixels()}, self.data() ? py::cast(self) : py::none{}); }, "View on pixel data"); } @@ -136,7 +137,7 @@ template void imageView(py::class_>& c) { pyObjectHolderFor(data).owner; }, "Image data") .def_property_readonly("pixels", [](T& self) { - return Containers::pyArrayViewHolder(self.pixels(), pyObjectHolderFor(self).owner); + return Containers::pyArrayViewHolder(Containers::PyStridedArrayView{self.pixels()}, pyObjectHolderFor(self).owner); }, "View on pixel data") .def_property_readonly("owner", [](T& self) { diff --git a/src/python/magnum/trade.cpp b/src/python/magnum/trade.cpp index 9b0d5fd..23f839e 100644 --- a/src/python/magnum/trade.cpp +++ b/src/python/magnum/trade.cpp @@ -32,6 +32,7 @@ #include #include "Corrade/Containers/PythonBindings.h" +#include "Corrade/Containers/StridedArrayViewPythonBindings.h" #include "Magnum/PythonBindings.h" #include "corrade/pluginmanager.h" @@ -132,7 +133,7 @@ template void imageData(py::class_{self.pixels()}, py::cast(self)); }, "View on pixel data"); }