diff --git a/doc/changelog.dox b/doc/changelog.dox index 55b1922d7..7e3df1d37 100644 --- a/doc/changelog.dox +++ b/doc/changelog.dox @@ -67,6 +67,8 @@ See also: - New @ref DebugTools::screenshot() function for convenient saving of screenshots +- @ref DebugTools::CompareImage and variants now properly handle + floating-point specials such as NaN or @f$ \infty @f$ in compared data @subsubsection changelog-latest-new-gl GL library diff --git a/src/Magnum/DebugTools/CompareImage.cpp b/src/Magnum/DebugTools/CompareImage.cpp index a246c3531..ab3d634f6 100644 --- a/src/Magnum/DebugTools/CompareImage.cpp +++ b/src/Magnum/DebugTools/CompareImage.cpp @@ -68,9 +68,25 @@ template Float calculateImageDelta(const ImageView2D& Math::Vector actualPixel{pixelAt(actualPixels, actualStride, {Int(x), Int(y)})}; Math::Vector expectedPixel{pixelAt(expectedPixels, expectedStride, {Int(x), Int(y)})}; - const Float value = (Math::abs(actualPixel - expectedPixel)).sum()/size; - output[y*expected.size().x() + x] = value; - max = Math::max(max, value); + /* First calculate a classic difference */ + Math::Vector diff = Math::abs(actualPixel - expectedPixel); + + /* Mark pixels that are NaN in both actual and expected pixels as + having no difference */ + diff = Math::lerp(diff, {}, Math::isNan(actualPixel) & Math::isNan(expectedPixel)); + + /* Then also mark pixels that are the same sign of infnity in both + actual and expected pixel as having no difference */ + diff = Math::lerp(diff, {}, Math::isInf(actualPixel) & Math::isInf(expectedPixel) & Math::equal(actualPixel, expectedPixel)); + + /* Calculate the difference and save it to the output image even + with NaN and ±Inf (as the user should know) */ + output[y*expected.size().x() + x] = diff.sum()/size; + + /* On the other hand, infs and NaNs should not contribute to the + max delta -- because all other differences would be zero + compared to them */ + max = Math::max(max, Math::lerp(diff, {}, Math::isNan(diff)|Math::isInf(diff)).sum()/size); } } @@ -142,7 +158,10 @@ std::tuple, Float, Float> calculateImageDelta(const Ima "DebugTools::CompareImage: unknown format" << expected.format(), {}); /* Calculate mean delta. Do it the special way so we don't lose - precision -- that would result in having false negatives! */ + precision -- that would result in having false negatives! This + *deliberately* leaves specials in. The `max` has them already filtered + out so if this would filter them out as well, there would be nothing + left that could cause the comparison to fail. */ const Float mean = Math::Algorithms::kahanSum(delta.begin(), delta.end())/delta.size(); return std::make_tuple(std::move(delta), max, mean); @@ -151,7 +170,8 @@ std::tuple, Float, Float> calculateImageDelta(const Ima namespace { /* Done by printing an white to black gradient using one of the online ASCII converters. Yes, I'm lazy. Another one could be " .,:;ox%#@". */ - const char Characters[] = " .,:~=+?7IZ$08DNM"; + constexpr char CharacterData[] = " .,:~=+?7IZ$08DNM"; + constexpr Containers::ArrayView Characters{CharacterData, sizeof(CharacterData) - 1}; } void printDeltaImage(Debug& out, Containers::ArrayView deltas, const Vector2i& size, const Float max, const Float maxThreshold, const Float meanThreshold) { @@ -171,11 +191,18 @@ void printDeltaImage(Debug& out, Containers::ArrayView deltas, cons const Vector2i blockSize = Math::min(size - offset, Vector2i{pixelsPerBlock}); Float blockMax{}; - for(std::int_fast32_t yb = 0; yb != blockSize.y(); ++yb) - for(std::int_fast32_t xb = 0; xb != blockSize.x(); ++xb) - blockMax = Math::max(blockMax, deltas[(offset.y() + yb)*size.x() + offset.x() + xb]); - - const char c = Characters[Int(Math::round(Math::min(blockMax/max, 1.0f)*(sizeof(Characters) - 2)))]; + for(std::int_fast32_t yb = 0; yb != blockSize.y(); ++yb) { + for(std::int_fast32_t xb = 0; xb != blockSize.x(); ++xb) { + /* Propagating NaNs. The delta should never be negative -- + but we need to test inversely in order to work correctly + for NaNs. */ + const Float delta = deltas[(offset.y() + yb)*size.x() + offset.x() + xb]; + CORRADE_INTERNAL_ASSERT(!(delta < 0.0f)); + blockMax = Math::max(delta, blockMax); + } + } + + const char c = Math::isNan(blockMax) ? Characters.back() : Characters[Int(Math::round(Math::min(1.0f, blockMax/max)*(Characters.size() - 1)))]; if(blockMax > maxThreshold) out << Debug::boldColor(Debug::Color::Red) << Debug::nospace << std::string{c} << Debug::resetColor; @@ -267,10 +294,10 @@ void printPixelDeltas(Debug& out, Containers::ArrayView delta, cons const std::size_t expectedStride = size.x(); /* Find first maxCount values above mean threshold and put them into a - sorted map */ + sorted map. Need to reverse the condition in order to catch NaNs. */ std::multimap large; for(std::size_t i = 0; i != delta.size(); ++i) - if(delta[i] > meanThreshold) large.emplace(delta[i], i); + if(!(delta[i] <= meanThreshold)) large.emplace(delta[i], i); CORRADE_INTERNAL_ASSERT(!large.empty()); @@ -336,6 +363,9 @@ class ImageComparatorBase::FileState { ImageComparatorBase::ImageComparatorBase(PluginManager::Manager* manager, Float maxThreshold, Float meanThreshold): _maxThreshold{maxThreshold}, _meanThreshold{meanThreshold}, _max{}, _mean{} { if(manager) _fileState.reset(new FileState{*manager}); + CORRADE_ASSERT(!Math::isNan(maxThreshold) && !Math::isInf(maxThreshold) && + !Math::isNan(meanThreshold) && !Math::isInf(meanThreshold), + "DebugTools::CompareImage: thresholds can't be NaN or infinity", ); CORRADE_ASSERT(meanThreshold <= maxThreshold, "DebugTools::CompareImage: maxThreshold can't be smaller than meanThreshold", ); } @@ -359,12 +389,20 @@ bool ImageComparatorBase::operator()(const ImageView2D& actual, const ImageView2 Containers::Array delta; std::tie(delta, _max, _mean) = DebugTools::Implementation::calculateImageDelta(actual, expected); - /* If both values are not above threshold, success */ - if(_max > _maxThreshold && _mean > _meanThreshold) + /* Verify the max/mean is never below zero so we didn't mess up when + calculating specials. Note the inverted condition to catch NaNs in + _mean. The max should OTOH be never special as it would make all other + deltas become zero in comparison. */ + CORRADE_INTERNAL_ASSERT(!(_mean < 0.0f)); + CORRADE_INTERNAL_ASSERT(_max >= 0.0f && !Math::isInf(_max) && !Math::isNan(_max)); + + /* If both values are not above threshold, success. Comparing this way in + order to properly catch NaNs in mean values. */ + if(_max > _maxThreshold && !(_mean <= _meanThreshold)) _state = State::AboveThresholds; else if(_max > _maxThreshold) _state = State::AboveMaxThreshold; - else if(_mean > _meanThreshold) + else if(!(_mean <= _meanThreshold)) _state = State::AboveMeanThreshold; else return true; diff --git a/src/Magnum/DebugTools/CompareImage.h b/src/Magnum/DebugTools/CompareImage.h index af788361d..218ac06b7 100644 --- a/src/Magnum/DebugTools/CompareImage.h +++ b/src/Magnum/DebugTools/CompareImage.h @@ -206,6 +206,30 @@ printed as characters of different perceived brightness. Blocks with delta over the max threshold are colored red, blocks with delta over the mean threshold are colored yellow. The delta list contains X,Y pixel position (with origin at bottom left), actual and expected pixel value and calculated delta. + +@section DebugTools-CompareImage-specials Special floating-point values + +For floating-point input, the comparator treats the values similarly to how +@ref Corrade::TestSuite::Comparator behaves for scalars: + +- If both actual and expected channel value are NaN, they are treated as the + same (with channel delta being 0). +- If actual and expected channel value have the same sign of infinity, they + are treated the same (with channel delta being 0). +- Otherwise, the delta is calculated the usual way, with NaN and infinity + values getting propagated according to floating-point rules. This means + the final per-pixel @f$ \Delta_{\boldsymbol{p}} @f$ becomes either NaN or + infinity. +- When calculating the max value, NaN and infinity @f$ \Delta_{\boldsymbol{p}} @f$ + values are ignored. This is done in order to avoid a single infinity deltas + causing all other deltas to be comparatively zero in the ASCII-art + representation. +- The mean value is calculated as usual, meaning that NaN or infinity in + @f$ \Delta_{\boldsymbol{p}} @f$ "poison" the final value, reliably causing + the comparison to fail. + +For the ASCII-art representation, NaN and infinity @f$ \Delta_{\boldsymbol{p}} @f$ +values are always treated as maximum difference. */ class CompareImage { public: diff --git a/src/Magnum/DebugTools/Test/CompareImageTest.cpp b/src/Magnum/DebugTools/Test/CompareImageTest.cpp index 7921c4911..74e77e660 100644 --- a/src/Magnum/DebugTools/Test/CompareImageTest.cpp +++ b/src/Magnum/DebugTools/Test/CompareImageTest.cpp @@ -53,13 +53,17 @@ struct CompareImageTest: TestSuite::Tester { void calculateDelta(); void calculateDeltaStorage(); + void calculateDeltaSpecials(); + void calculateDeltaSpecials3(); void deltaImage(); void deltaImageScaling(); void deltaImageColors(); + void deltaImageSpecials(); void pixelDelta(); void pixelDeltaOverflow(); + void pixelDeltaSpecials(); void compareDifferentSize(); void compareDifferentFormat(); @@ -67,6 +71,9 @@ struct CompareImageTest: TestSuite::Tester { void compareAboveThresholds(); void compareAboveMaxThreshold(); void compareAboveMeanThreshold(); + void compareSpecials(); + void compareSpecialsMeanOnly(); + void compareSpecialsDisallowedThreshold(); void setupExternalPluginManager(); void teardownExternalPluginManager(); @@ -102,13 +109,17 @@ CompareImageTest::CompareImageTest() { &CompareImageTest::calculateDelta, &CompareImageTest::calculateDeltaStorage, + &CompareImageTest::calculateDeltaSpecials, + &CompareImageTest::calculateDeltaSpecials3, &CompareImageTest::deltaImage, &CompareImageTest::deltaImageScaling, &CompareImageTest::deltaImageColors, + &CompareImageTest::deltaImageSpecials, &CompareImageTest::pixelDelta, &CompareImageTest::pixelDeltaOverflow, + &CompareImageTest::pixelDeltaSpecials, &CompareImageTest::compareDifferentSize, &CompareImageTest::compareDifferentFormat, @@ -116,6 +127,9 @@ CompareImageTest::CompareImageTest() { &CompareImageTest::compareAboveThresholds, &CompareImageTest::compareAboveMaxThreshold, &CompareImageTest::compareAboveMeanThreshold, + &CompareImageTest::compareSpecials, + &CompareImageTest::compareSpecialsMeanOnly, + &CompareImageTest::compareSpecialsDisallowedThreshold, &CompareImageTest::image, &CompareImageTest::imageError}); @@ -254,6 +268,72 @@ void CompareImageTest::calculateDeltaStorage() { CORRADE_COMPARE(mean, 18.5f); } +/* Variants: + - expected number, got inf (and inverse) + - expected number, got nan (and inverse) + - got inf in both (and again, but different sign) + - got nan in both + - got a number in both (twice, to ensure it's calculated correctly) */ +const Float ActualDataSpecials[]{ + Constants::inf(), 0.3f, + Constants::nan(), 0.3f, + -Constants::inf(), -Constants::inf(), + Constants::nan(), + 0.3f, 3.0f +}; +const Float ExpectedDataSpecials[]{ + 1.0f, -Constants::inf(), + 0.3f, Constants::nan(), + -Constants::inf(), Constants::inf(), + Constants::nan(), + 0.65f, -0.1f +}; +const Float DeltaSpecials[]{ + Constants::inf(), Constants::inf(), + Constants::nan(), Constants::nan(), + 0.0f, Constants::inf(), + 0.0f, + 0.35f, 3.1f +}; + +const ImageView2D ActualSpecials{PixelFormat::R32F, {9, 1}, ActualDataSpecials}; +const ImageView2D ExpectedSpecials{PixelFormat::R32F, {9, 1}, ExpectedDataSpecials}; + +void CompareImageTest::calculateDeltaSpecials() { + Containers::Array delta; + Float max, mean; + std::tie(delta, max, mean) = Implementation::calculateImageDelta(ActualSpecials, ExpectedSpecials); + CORRADE_COMPARE_AS(Containers::arrayView(delta), + Containers::arrayView(DeltaSpecials), + TestSuite::Compare::Container); + /* Max should be calculated *without* the specials because otherwise every + other potential difference will be zero compared to infinity. OTOH mean + needs to get "poisoned" by those in order to have *something* to fail + the test with. */ + CORRADE_COMPARE(max, 3.1f); + CORRADE_COMPARE(mean, -Constants::nan()); +} + +void CompareImageTest::calculateDeltaSpecials3() { + /* Same as calculateDeltaSpecials(), but reinterpreting the data as + a three-component vector in order to test per-component handling of + specials */ + const ImageView2D actualSpecials3{PixelFormat::RGB32F, {3, 1}, ActualDataSpecials}; + const ImageView2D expectedSpecials3{PixelFormat::RGB32F, {3, 1}, ExpectedDataSpecials}; + + Containers::Array delta; + Float max, mean; + std::tie(delta, max, mean) = Implementation::calculateImageDelta(actualSpecials3, expectedSpecials3); + CORRADE_COMPARE_AS(delta, (Containers::Array{Containers::InPlaceInit, { + Constants::nan(), Constants::nan(), 1.15f + }}), TestSuite::Compare::Container); + /* Max and mean should be calculated *without* the specials because + otherwise every other potential difference will be zero compared to + infinity */ + CORRADE_COMPARE(max, 1.15f); + CORRADE_COMPARE(mean, -Constants::nan()); +} + void CompareImageTest::deltaImage() { std::ostringstream out; Debug d{&out, Debug::Flag::DisableColors}; @@ -325,6 +405,25 @@ void CompareImageTest::deltaImageColors() { " |: ,|\n"); } +void CompareImageTest::deltaImageSpecials() { + const Float inf = Constants::inf(), nan = Constants::nan(); + /* Duplicate the rows as the delta image visualizer merges each two to + preserve ratio */ + const Float delta[]{0.7f, inf, 2.5f, + 0.7f, nan, 2.5f, + nan, inf, 0.0f, + nan, inf, 0.0f}; + + std::ostringstream out; + Debug dc{&out, Debug::Flag::DisableColors}; + Implementation::printDeltaImage(dc, delta, {3, 4}, 3.0f, 0.0f, 0.0f); + /* Should show the max value for NaN and infs and the usual things + otherwise */ + CORRADE_COMPARE(out.str(), + " |MM |\n" + " |~M8|\n"); +} + void CompareImageTest::pixelDelta() { { Debug() << "Visual verification -- some lines should be yellow, some red:"; @@ -356,6 +455,33 @@ void CompareImageTest::pixelDeltaOverflow() { " [2,0] Vector(0.9), expected Vector(0.6) (Δ = 0.3)"); } +void CompareImageTest::pixelDeltaSpecials() { + std::ostringstream out; + Debug d{&out, Debug::Flag::DisableColors}; + Implementation::printPixelDeltas(d, DeltaSpecials, ActualSpecials, ExpectedSpecials, 1.5f, 0.5f, 10); + + /* MSVC prints -nan(ind) instead of ±nan. But only sometimes. */ + #ifdef _MSC_VER + CORRADE_COMPARE(out.str(), + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(-nan(ind)) (Δ = -nan(ind))\n" + " [2,0] Vector(-nan(ind)), expected Vector(0.3) (Δ = -nan(ind))\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)"); + #else + CORRADE_COMPARE(out.str(), + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n" + " [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)"); + #endif +} + void CompareImageTest::compareDifferentSize() { std::stringstream out; @@ -454,6 +580,124 @@ void CompareImageTest::compareAboveMeanThreshold() { " [1,0] #5647ec, expected #5610ed (Δ = 18.6667)\n"); } +void CompareImageTest::compareSpecials() { + std::stringstream out; + + { + TestSuite::Comparator compare{1.5f, 0.5f}; + CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials)); + Debug d{&out, Debug::Flag::DisableColors}; + compare.printErrorMessage(d, "a", "b"); + } + + /* Apple platforms, Android, Emscripten and MinGW32 don't print signed + NaNs. This is *not* a libc++ thing, tho -- libc++ on Linux prints signed NaNs. */ + #if defined(CORRADE_TARGET_APPLE) || defined(CORRADE_TARGET_ANDROID) || defined(CORRADE_TARGET_EMSCRIPTEN) || defined(__MINGW32__) + CORRADE_COMPARE(out.str(), + "Images a and b have both max and mean delta above threshold, actual 3.1/nan but at most 1.5/0.5 expected. Delta image:\n" + " |MMMM M ,M|\n" + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n" + " [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n"); + + /* MSVC prints -nan(ind) instead of ±nan. But only sometimes. */ + #elif defined(_MSC_VER) + CORRADE_COMPARE(out.str(), + "Images a and b have both max and mean delta above threshold, actual 3.1/-nan(ind) but at most 1.5/0.5 expected. Delta image:\n" + " |MMMM M ,M|\n" + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(-nan(ind)) (Δ = nan)\n" + " [2,0] Vector(-nan(ind)), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n"); + + /* Linux */ + #else + CORRADE_COMPARE(out.str(), + "Images a and b have both max and mean delta above threshold, actual 3.1/-nan but at most 1.5/0.5 expected. Delta image:\n" + " |MMMM M ,M|\n" + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n" + " [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n"); + #endif +} + +void CompareImageTest::compareSpecialsMeanOnly() { + std::stringstream out; + + { + TestSuite::Comparator compare{15.0f, 0.5f}; + CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials)); + Debug d{&out, Debug::Flag::DisableColors}; + compare.printErrorMessage(d, "a", "b"); + } + + /* Apple platforms, Android, Emscripten and MinGW32 don't print signed + NaNs. This is *not* a libc++ thing, tho -- libc++ on Linux prints signed NaNs. */ + #if defined(CORRADE_TARGET_APPLE) || defined(CORRADE_TARGET_ANDROID) || defined(CORRADE_TARGET_EMSCRIPTEN) || defined(__MINGW32__) + CORRADE_COMPARE(out.str(), + "Images a and b have mean delta above threshold, actual nan but at most 0.5 expected. Max delta 3.1 is below threshold 15. Delta image:\n" + " |MMMM M ,M|\n" + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n" + " [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n"); + + /* MSVC prints -nan(ind) instead of ±nan. But only sometimes. */ + #elif defined(CORRADE_TARGET_WINDOWS) && defined(_MSC_VER) + CORRADE_COMPARE(out.str(), + "Images a and b have mean delta above threshold, actual -nan(ind) but at most 0.5 expected. Max delta 3.1 is below threshold 15. Delta image:\n" + " |MMMM M ,M|\n" + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(-nan(ind)) (Δ = nan)\n" + " [2,0] Vector(-nan(ind)), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n"); + + /* Linux */ + #else + CORRADE_COMPARE(out.str(), + "Images a and b have mean delta above threshold, actual -nan but at most 0.5 expected. Max delta 3.1 is below threshold 15. Delta image:\n" + " |MMMM M ,M|\n" + " Pixels above max/mean threshold:\n" + " [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n" + " [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n" + " [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n" + " [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n" + " [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n" + " [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n"); + #endif +} + +void CompareImageTest::compareSpecialsDisallowedThreshold() { + std::stringstream out; + + { + Error redirectError{&out}; + TestSuite::Comparator a{Constants::inf(), 0.3f}; + TestSuite::Comparator b{0.3f, Constants::nan()}; + } + + CORRADE_COMPARE(out.str(), + "DebugTools::CompareImage: thresholds can't be NaN or infinity\n" + "DebugTools::CompareImage: thresholds can't be NaN or infinity\n"); +} + void CompareImageTest::setupExternalPluginManager() { _manager.emplace("nonexistent"); /* Load the plugin directly from the build tree. Otherwise it's either