MeshTools: disallow impl-spec index types in removeDuplicates*().

src/Magnum/MeshTools/RemoveDuplicates.cpp

@@ -409,6 +409,12 @@ Trade::MeshData removeDuplicates(const Trade::MeshData& data) {
     }
     #endif
 
+    /* This has to be checked before passing the data to interleave() as there
+       it would die also, but with a confusing function name in the message */
+    CORRADE_ASSERT(!data.isIndexed() || !isMeshIndexTypeImplementationSpecific(data.indexType()),
+        "MeshTools::removeDuplicates(): mesh has an implementation-specific index type" << reinterpret_cast<void*>(meshIndexTypeUnwrap(data.indexType())),
+        (Trade::MeshData{MeshPrimitive{}, 0}));
+
     /* Turn the passed data into an interleaved owned mutable instance we can
        operate on -- owned() alone only makes the data owned, interleave()
        alone only makes the data interleaved (but the index data can stay
@@ -580,6 +586,9 @@ Trade::MeshData removeDuplicatesFuzzy(const Trade::MeshData& data, const Float f
             Containers::arrayCast<UnsignedInt>(indexData));
         indexType = MeshIndexType::UnsignedInt;
     } else {
+        CORRADE_ASSERT(!isMeshIndexTypeImplementationSpecific(owned.indexType()),
+            "MeshTools::removeDuplicatesFuzzy(): mesh has an implementation-specific index type" << reinterpret_cast<void*>(meshIndexTypeUnwrap(owned.indexType())),
+            (Trade::MeshData{MeshPrimitive{}, 0}));
         vertexCount = removeDuplicatesIndexedInPlace(
             owned.mutableIndices(),
             Containers::arrayCast<2, char>(combinedIndices));

src/Magnum/MeshTools/RemoveDuplicates.h

@@ -299,13 +299,15 @@ newly generated index buffer into a new @ref Trade::MeshData instance. If the
 mesh is indexed, the original index type is preserved, otherwise the mesh gets
 @ref MeshIndexType::UnsignedInt indices. The resulting mesh is always
 interleaved and owned, if the input is already interleaved attribute offsets
-and paddings are preserved. All attributes are expected to not have an
+and paddings are preserved. An index buffer, if present, is expected to not
-implementation-specific format.
+have an implementation-specific index type. All attributes are expected to not
+have an implementation-specific format.
 
 In order to remove random padding values from the input and make the vertices
 suitable for fast in-place duplicate removal, this function unconditionally
 copies and interleaves the input vertex and index data.
-@see @ref isVertexFormatImplementationSpecific()
+@see @ref isMeshIndexTypeImplementationSpecific(),
+    @ref isVertexFormatImplementationSpecific()
 */
 MAGNUM_MESHTOOLS_EXPORT Trade::MeshData removeDuplicates(const Trade::MeshData& data);
 

src/Magnum/MeshTools/Test/RemoveDuplicatesTest.cpp

@@ -73,11 +73,13 @@ struct RemoveDuplicatesTest: TestSuite::Tester {
     void removeDuplicatesMeshData();
     void removeDuplicatesMeshDataPaddedAttributes();
     void removeDuplicatesMeshDataAttributeless();
+    void removeDuplicatesMeshDataImplementationSpecificIndexType();
     void removeDuplicatesMeshDataImplementationSpecificVertexFormat();
 
     void removeDuplicatesMeshDataFuzzy();
     void removeDuplicatesMeshDataFuzzyDouble();
     void removeDuplicatesMeshDataFuzzyAttributeless();
+    void removeDuplicatesMeshDataFuzzyImplementationSpecificIndexType();
     void removeDuplicatesMeshDataFuzzyImplementationSpecificVertexFormat();
 
     void soakTest();
@@ -213,6 +215,7 @@ RemoveDuplicatesTest::RemoveDuplicatesTest() {
     addTests({&RemoveDuplicatesTest::removeDuplicatesMeshDataPaddedAttributes});
 
     addTests({&RemoveDuplicatesTest::removeDuplicatesMeshDataAttributeless,
+              &RemoveDuplicatesTest::removeDuplicatesMeshDataImplementationSpecificIndexType,
               &RemoveDuplicatesTest::removeDuplicatesMeshDataImplementationSpecificVertexFormat});
 
     addInstancedTests({&RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzy},
@@ -221,6 +224,7 @@ RemoveDuplicatesTest::RemoveDuplicatesTest() {
     addTests({&RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyDouble,
 
               &RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyAttributeless,
+              &RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyImplementationSpecificIndexType,
               &RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyImplementationSpecificVertexFormat});
 
     addRepeatedTests({&RemoveDuplicatesTest::soakTest,
@@ -822,6 +826,22 @@ void RemoveDuplicatesTest::removeDuplicatesMeshDataAttributeless() {
         "MeshTools::removeDuplicates(): can't remove duplicates in an attributeless mesh\n");
 }
 
+void RemoveDuplicatesTest::removeDuplicatesMeshDataImplementationSpecificIndexType() {
+    #ifdef CORRADE_NO_ASSERT
+    CORRADE_SKIP("CORRADE_NO_ASSERT defined, can't test assertions");
+    #endif
+
+    std::ostringstream out;
+    Error redirectError{&out};
+    MeshTools::removeDuplicates(Trade::MeshData{MeshPrimitive::Points,
+        nullptr, Trade::MeshIndexData{meshIndexTypeWrap(0xcaca), Containers::StridedArrayView1D<const void>{}},
+        nullptr, {
+            Trade::MeshAttributeData{Trade::MeshAttribute::Position, VertexFormat::Vector3, nullptr}
+        }});
+    CORRADE_COMPARE(out.str(),
+        "MeshTools::removeDuplicates(): mesh has an implementation-specific index type 0xcaca\n");
+}
+
 void RemoveDuplicatesTest::removeDuplicatesMeshDataImplementationSpecificVertexFormat() {
     #ifdef CORRADE_NO_ASSERT
     CORRADE_SKIP("CORRADE_NO_ASSERT defined, can't test assertions");
@@ -1156,6 +1176,22 @@ void RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyAttributeless() {
         "MeshTools::removeDuplicatesFuzzy(): can't remove duplicates in an attributeless mesh\n");
 }
 
+void RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyImplementationSpecificIndexType() {
+    #ifdef CORRADE_NO_ASSERT
+    CORRADE_SKIP("CORRADE_NO_ASSERT defined, can't test assertions");
+    #endif
+
+    std::ostringstream out;
+    Error redirectError{&out};
+    MeshTools::removeDuplicatesFuzzy(Trade::MeshData{MeshPrimitive::Points,
+        nullptr, Trade::MeshIndexData{meshIndexTypeWrap(0xcaca), Containers::StridedArrayView1D<const void>{}},
+        nullptr, {
+            Trade::MeshAttributeData{Trade::MeshAttribute::Position, VertexFormat::Vector3, nullptr}
+        }});
+    CORRADE_COMPARE(out.str(),
+        "MeshTools::removeDuplicatesFuzzy(): mesh has an implementation-specific index type 0xcaca\n");
+}
+
 void RemoveDuplicatesTest::removeDuplicatesMeshDataFuzzyImplementationSpecificVertexFormat() {
     #ifdef CORRADE_NO_ASSERT
     CORRADE_SKIP("CORRADE_NO_ASSERT defined, can't test assertions");