/****************************************************************************** * Copyright (c) 2019, Hobu Inc. (info@hobu.co) * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following * conditions are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided * with the distribution. * * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the * names of its contributors may be used to endorse or promote * products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY * OF SUCH DAMAGE. ****************************************************************************/ #include "PyArray.hpp" #include namespace pdal { namespace python { namespace { Dimension::Type pdalType(int t) { using namespace Dimension; switch (t) { case NPY_FLOAT32: return Type::Float; case NPY_FLOAT64: return Type::Double; case NPY_INT8: return Type::Signed8; case NPY_INT16: return Type::Signed16; case NPY_INT32: return Type::Signed32; case NPY_INT64: return Type::Signed64; case NPY_UINT8: return Type::Unsigned8; case NPY_UINT16: return Type::Unsigned16; case NPY_UINT32: return Type::Unsigned32; case NPY_UINT64: return Type::Unsigned64; default: return Type::None; } assert(0); return Type::None; } std::string pyObjectToString(PyObject *pname) { PyObject* r = PyObject_Str(pname); if (!r) throw pdal_error("couldn't make string representation value"); Py_ssize_t size; return std::string(PyUnicode_AsUTF8AndSize(r, &size)); } } // unnamed namespace #if NPY_ABI_VERSION < 0x02000000 #define PyDataType_FIELDS(descr) ((descr)->fields) #define PyDataType_NAMES(descr) ((descr)->names) #endif Array::Array(PyArrayObject* array, std::shared_ptr stream_handler) : m_array(array), m_rowMajor(true), m_stream_handler(std::move(stream_handler)) { Py_XINCREF(array); PyArray_Descr *dtype = PyArray_DTYPE(m_array); npy_intp ndims = PyArray_NDIM(m_array); npy_intp *shape = PyArray_SHAPE(m_array); PyObject* fields = PyDataType_FIELDS(dtype); int numFields = (fields == Py_None) ? 0 : static_cast(PyDict_Size(fields)); int xyz = 0; if (numFields == 0) { if (ndims != 3) throw pdal_error("Array without fields must have 3 dimensions."); m_fields.push_back({"Intensity", pdalType(dtype->type_num), 0}); } else { PyObject *names_dict = fields; PyObject *names = PyDict_Keys(names_dict); PyObject *values = PyDict_Values(names_dict); if (!names || !values) throw pdal_error("Bad field specification in numpy array."); for (int i = 0; i < numFields; ++i) { std::string name = python::pyObjectToString(PyList_GetItem(names, i)); if (name == "X") xyz |= 1; else if (name == "Y") xyz |= 2; else if (name == "Z") xyz |= 4; PyObject *tup = PyList_GetItem(values, i); // Get offset. size_t offset = PyLong_AsLong(PySequence_Fast_GET_ITEM(tup, 1)); // Get type. PyArray_Descr *descriptor = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0); Dimension::Type type = pdalType(descriptor->type_num); if (type == Dimension::Type::None) throw pdal_error("Incompatible type for field '" + name + "'."); m_fields.push_back({name, type, offset}); } if (xyz != 0 && xyz != 7) throw pdal_error("Array fields must contain all or none " "of X, Y and Z"); if (xyz == 0 && ndims != 3) throw pdal_error("Array without named X/Y/Z fields " "must have three dimensions."); } if (xyz == 0) m_shape = { (size_t)shape[0], (size_t)shape[1], (size_t)shape[2] }; m_rowMajor = !(PyArray_FLAGS(m_array) & NPY_ARRAY_F_CONTIGUOUS); } Array::~Array() { Py_XDECREF(m_array); } std::shared_ptr Array::iterator() { return std::make_shared(m_array, m_stream_handler); } ArrayIter::ArrayIter(PyArrayObject* np_array, std::shared_ptr stream_handler) : m_stream_handler(std::move(stream_handler)) { // Create iterator m_iter = NpyIter_New(np_array, NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, NPY_KEEPORDER, NPY_NO_CASTING, NULL); if (!m_iter) throw pdal_error("Unable to create numpy iterator."); initIterator(); } void ArrayIter::initIterator() { // For a stream handler, first execute it to get the buffer populated and know the size of the data to iterate int64_t stream_chunk_size = 0; if (m_stream_handler) { stream_chunk_size = (*m_stream_handler)(); if (!stream_chunk_size) { m_done = true; return; } } // Initialize the iterator function char *itererr; m_iterNext = NpyIter_GetIterNext(m_iter, &itererr); if (!m_iterNext) { NpyIter_Deallocate(m_iter); m_iter = nullptr; throw pdal_error(std::string("Unable to retrieve iteration function from numpy iterator: ") + itererr); } m_data = NpyIter_GetDataPtrArray(m_iter); m_stride = *NpyIter_GetInnerStrideArray(m_iter); m_size = *NpyIter_GetInnerLoopSizePtr(m_iter); if (stream_chunk_size) { // Ensure chunk size is valid and then limit iteration accordingly if (0 < stream_chunk_size && stream_chunk_size <= m_size) { m_size = stream_chunk_size; } else { throw pdal_error(std::string("Stream chunk size not in the range of array length: ") + std::to_string(stream_chunk_size)); } } m_done = false; } void ArrayIter::resetIterator() { // Reset the iterator to the initial state if (NpyIter_Reset(m_iter, NULL) != NPY_SUCCEED) { NpyIter_Deallocate(m_iter); m_iter = nullptr; throw pdal_error("Unable to reset numpy iterator."); } initIterator(); } ArrayIter::~ArrayIter() { if (m_iter != nullptr) { NpyIter_Deallocate(m_iter); } } ArrayIter& ArrayIter::operator++() { if (m_done) return *this; if (--m_size) { *m_data += m_stride; } else if (!m_iterNext(m_iter)) { if (m_stream_handler) { resetIterator(); } else { m_done = true; } } return *this; } } // namespace python } // namespace pdal