MeshTools: support an extreme corner case in interleavedData().

I need this for combineFaceAttributes() internals, I don't expect this
to be useful for actual humans.

src/Magnum/MeshTools/Interleave.cpp

@@ -25,6 +25,7 @@
 
 #include "Interleave.h"
 
+#include <Corrade/Containers/Optional.h>
 #include <Corrade/Utility/Algorithms.h>
 
 #include "Magnum/Math/Functions.h"
@@ -34,17 +35,16 @@ namespace Magnum { namespace MeshTools {
 
 namespace {
 
-Containers::StridedArrayView2D<const char> interleavedDataInternal(const Trade::MeshData& data) {
+Containers::Optional<Containers::StridedArrayView2D<const char>> interleavedDataInternal(const Trade::MeshData& data) {
-    /* There is no attributes, return a non-nullptr zero-sized view to indicate
+    /* There is no attributes, return a zero-sized view to indicate a success */
-       a success */
+    if(!data.attributeCount())
-    if(!data.attributeCount() || !data.vertexData())
         return Containers::StridedArrayView2D<const char>{data.vertexData(), {data.vertexCount(), 0}};
 
     const UnsignedInt stride = data.attributeStride(0);
     std::size_t minOffset = data.attributeOffset(0);
     std::size_t maxOffset = minOffset + vertexFormatSize(data.attributeFormat(0));
     for(UnsignedInt i = 0; i != data.attributeCount(); ++i) {
-        if(data.attributeStride(i) != stride) return nullptr;
+        if(data.attributeStride(i) != stride) return Containers::NullOpt;
 
         const std::size_t offset = data.attributeOffset(i);
         minOffset = Math::min(minOffset, offset);
@@ -52,7 +52,7 @@ Containers::StridedArrayView2D<const char> interleavedDataInternal(const Trade::
     }
 
     /* The offsets can't fit into the stride, report failure */
-    if(maxOffset - minOffset > stride) return nullptr;
+    if(maxOffset - minOffset > stride) return Containers::NullOpt;
 
     return Containers::StridedArrayView2D<const char>{
         data.vertexData(), data.vertexData().data() + minOffset,
@@ -63,16 +63,13 @@ Containers::StridedArrayView2D<const char> interleavedDataInternal(const Trade::
 }
 
 bool isInterleaved(const Trade::MeshData& data) {
-    /* If a nullptr value is returned but the mesh vertex data is not nullptr,
+    return !!interleavedDataInternal(data);
-       the mesh is not interleaved. Otherwise it is */
-    return !!interleavedDataInternal(data).data() == !!data.vertexData().data();
 }
 
 Containers::StridedArrayView2D<const char> interleavedData(const Trade::MeshData& data) {
     auto out = interleavedDataInternal(data);
-    CORRADE_ASSERT(out || !data.vertexData(),
+    CORRADE_ASSERT(out, "MeshTools::interleavedData(): the mesh is not interleaved", {});
-        "MeshTools::interleavedData(): the mesh is not interleaved", {});
+    return *out;
-    return out;
 }
 
 namespace Implementation {

src/Magnum/MeshTools/Test/InterleaveTest.cpp

@@ -57,11 +57,13 @@ struct InterleaveTest: Corrade::TestSuite::Tester {
     void isInterleavedAliased();
     void isInterleavedUnordered();
     void isInterleavedAttributeAcrossStride();
+    void isInterleavedVertexDataWholeMemory();
 
     void interleavedData();
     void interleavedDataNoAttributes();
     void interleavedDataNoVertices();
     void interleavedDataNotInterleaved();
+    void interleavedDataVertexDataWholeMemory();
 
     void interleavedLayout();
     void interleavedLayoutExtra();
@@ -104,11 +106,13 @@ InterleaveTest::InterleaveTest() {
               &InterleaveTest::isInterleavedAliased,
               &InterleaveTest::isInterleavedUnordered,
               &InterleaveTest::isInterleavedAttributeAcrossStride,
+              &InterleaveTest::isInterleavedVertexDataWholeMemory,
 
               &InterleaveTest::interleavedData,
               &InterleaveTest::interleavedDataNoAttributes,
               &InterleaveTest::interleavedDataNoVertices,
               &InterleaveTest::interleavedDataNotInterleaved,
+              &InterleaveTest::interleavedDataVertexDataWholeMemory,
 
               &InterleaveTest::interleavedLayout,
               &InterleaveTest::interleavedLayoutExtra,
@@ -347,6 +351,25 @@ void InterleaveTest::isInterleavedAttributeAcrossStride() {
     CORRADE_VERIFY(!MeshTools::isInterleaved(data2));
 }
 
+void InterleaveTest::isInterleavedVertexDataWholeMemory() {
+    struct Vertex {
+        Vector2 position;
+        Vector3 normal;
+    } vertexData[3];
+    Trade::MeshAttributeData positions{Trade::MeshAttribute::Position,
+        Containers::StridedArrayView1D<Vector2>{vertexData,
+            &vertexData[0].position, 3, sizeof(Vertex)}};
+    Trade::MeshAttributeData normals{Trade::MeshAttribute::Normal,
+        Containers::StridedArrayView1D<Vector3>{vertexData,
+            &vertexData[0].normal, 3, sizeof(Vertex)}};
+
+    /* This is used internally by combineFaceAttributes(), as long as the
+       vertex data array isn't accessed directly it's okay */
+    Trade::MeshData data{MeshPrimitive::Triangles,
+        {}, {nullptr, ~std::size_t{}}, {positions, normals}};
+    CORRADE_VERIFY(MeshTools::isInterleaved(data));
+}
+
 void InterleaveTest::interleavedData() {
     Containers::Array<char> vertexData{100 + 3*40};
     Containers::StridedArrayView1D<Vector3> normals{vertexData,
@@ -417,6 +440,37 @@ void InterleaveTest::interleavedDataNotInterleaved() {
     CORRADE_COMPARE(out.str(), "MeshTools::interleavedData(): the mesh is not interleaved\n");
 }
 
+void InterleaveTest::interleavedDataVertexDataWholeMemory() {
+    struct Vertex {
+        int:32;
+        Vector2 position;
+        int:32;
+        int:32;
+        Vector3 normal;
+        int:32;
+        int:32;
+    } vertexData[3];
+    Trade::MeshAttributeData positions{Trade::MeshAttribute::Position,
+        Containers::StridedArrayView1D<Vector2>{vertexData,
+            &vertexData[0].position, 3, sizeof(Vertex)}};
+    Trade::MeshAttributeData normals{Trade::MeshAttribute::Normal,
+        Containers::StridedArrayView1D<Vector3>{vertexData,
+            &vertexData[0].normal, 3, sizeof(Vertex)}};
+
+    /* This is used internally by combineFaceAttributes(), as long as the
+       vertex data array isn't accessed directly it's okay */
+    Trade::MeshData data{MeshPrimitive::Triangles,
+        {}, {nullptr, ~std::size_t{}}, {normals, positions}};
+
+    CORRADE_VERIFY(MeshTools::isInterleaved(data));
+    Containers::StridedArrayView2D<const char> interleaved = MeshTools::interleavedData(data);
+    CORRADE_COMPARE(interleaved.data(), positions.data().data());
+    CORRADE_COMPARE(interleaved.size()[0], 3);
+    CORRADE_COMPARE(interleaved.size()[1], 28);
+    CORRADE_COMPARE(interleaved.stride()[0], 40);
+    CORRADE_COMPARE(interleaved.stride()[1], 1);
+}
+
 void InterleaveTest::interleavedLayout() {
     Containers::Array<char> indexData{6};
     Containers::Array<char> vertexData{3*20};