Browse Source

DebugTools: make CompareImage work correctly with NaNs and infinities.

pull/364/head
Vladimír Vondruš 7 years ago
parent
commit
5d9fb386f5
  1. 2
      doc/changelog.dox
  2. 68
      src/Magnum/DebugTools/CompareImage.cpp
  3. 24
      src/Magnum/DebugTools/CompareImage.h
  4. 244
      src/Magnum/DebugTools/Test/CompareImageTest.cpp

2
doc/changelog.dox

@ -67,6 +67,8 @@ See also:
- New @ref DebugTools::screenshot() function for convenient saving of
screenshots
- @ref DebugTools::CompareImage and variants now properly handle
floating-point specials such as NaN or @f$ \infty @f$ in compared data
@subsubsection changelog-latest-new-gl GL library

68
src/Magnum/DebugTools/CompareImage.cpp

@ -68,9 +68,25 @@ template<std::size_t size, class T> Float calculateImageDelta(const ImageView2D&
Math::Vector<size, Float> actualPixel{pixelAt<size, T>(actualPixels, actualStride, {Int(x), Int(y)})};
Math::Vector<size, Float> expectedPixel{pixelAt<size, T>(expectedPixels, expectedStride, {Int(x), Int(y)})};
const Float value = (Math::abs(actualPixel - expectedPixel)).sum()/size;
output[y*expected.size().x() + x] = value;
max = Math::max(max, value);
/* First calculate a classic difference */
Math::Vector<size, Float> diff = Math::abs(actualPixel - expectedPixel);
/* Mark pixels that are NaN in both actual and expected pixels as
having no difference */
diff = Math::lerp(diff, {}, Math::isNan(actualPixel) & Math::isNan(expectedPixel));
/* Then also mark pixels that are the same sign of infnity in both
actual and expected pixel as having no difference */
diff = Math::lerp(diff, {}, Math::isInf(actualPixel) & Math::isInf(expectedPixel) & Math::equal(actualPixel, expectedPixel));
/* Calculate the difference and save it to the output image even
with NaN and ±Inf (as the user should know) */
output[y*expected.size().x() + x] = diff.sum()/size;
/* On the other hand, infs and NaNs should not contribute to the
max delta -- because all other differences would be zero
compared to them */
max = Math::max(max, Math::lerp(diff, {}, Math::isNan(diff)|Math::isInf(diff)).sum()/size);
}
}
@ -142,7 +158,10 @@ std::tuple<Containers::Array<Float>, Float, Float> calculateImageDelta(const Ima
"DebugTools::CompareImage: unknown format" << expected.format(), {});
/* Calculate mean delta. Do it the special way so we don't lose
precision -- that would result in having false negatives! */
precision -- that would result in having false negatives! This
*deliberately* leaves specials in. The `max` has them already filtered
out so if this would filter them out as well, there would be nothing
left that could cause the comparison to fail. */
const Float mean = Math::Algorithms::kahanSum(delta.begin(), delta.end())/delta.size();
return std::make_tuple(std::move(delta), max, mean);
@ -151,7 +170,8 @@ std::tuple<Containers::Array<Float>, Float, Float> calculateImageDelta(const Ima
namespace {
/* Done by printing an white to black gradient using one of the online
ASCII converters. Yes, I'm lazy. Another one could be " .,:;ox%#@". */
const char Characters[] = " .,:~=+?7IZ$08DNM";
constexpr char CharacterData[] = " .,:~=+?7IZ$08DNM";
constexpr Containers::ArrayView<const char> Characters{CharacterData, sizeof(CharacterData) - 1};
}
void printDeltaImage(Debug& out, Containers::ArrayView<const Float> deltas, const Vector2i& size, const Float max, const Float maxThreshold, const Float meanThreshold) {
@ -171,11 +191,18 @@ void printDeltaImage(Debug& out, Containers::ArrayView<const Float> deltas, cons
const Vector2i blockSize = Math::min(size - offset, Vector2i{pixelsPerBlock});
Float blockMax{};
for(std::int_fast32_t yb = 0; yb != blockSize.y(); ++yb)
for(std::int_fast32_t xb = 0; xb != blockSize.x(); ++xb)
blockMax = Math::max(blockMax, deltas[(offset.y() + yb)*size.x() + offset.x() + xb]);
const char c = Characters[Int(Math::round(Math::min(blockMax/max, 1.0f)*(sizeof(Characters) - 2)))];
for(std::int_fast32_t yb = 0; yb != blockSize.y(); ++yb) {
for(std::int_fast32_t xb = 0; xb != blockSize.x(); ++xb) {
/* Propagating NaNs. The delta should never be negative --
but we need to test inversely in order to work correctly
for NaNs. */
const Float delta = deltas[(offset.y() + yb)*size.x() + offset.x() + xb];
CORRADE_INTERNAL_ASSERT(!(delta < 0.0f));
blockMax = Math::max(delta, blockMax);
}
}
const char c = Math::isNan(blockMax) ? Characters.back() : Characters[Int(Math::round(Math::min(1.0f, blockMax/max)*(Characters.size() - 1)))];
if(blockMax > maxThreshold)
out << Debug::boldColor(Debug::Color::Red) << Debug::nospace << std::string{c} << Debug::resetColor;
@ -267,10 +294,10 @@ void printPixelDeltas(Debug& out, Containers::ArrayView<const Float> delta, cons
const std::size_t expectedStride = size.x();
/* Find first maxCount values above mean threshold and put them into a
sorted map */
sorted map. Need to reverse the condition in order to catch NaNs. */
std::multimap<Float, std::size_t> large;
for(std::size_t i = 0; i != delta.size(); ++i)
if(delta[i] > meanThreshold) large.emplace(delta[i], i);
if(!(delta[i] <= meanThreshold)) large.emplace(delta[i], i);
CORRADE_INTERNAL_ASSERT(!large.empty());
@ -336,6 +363,9 @@ class ImageComparatorBase::FileState {
ImageComparatorBase::ImageComparatorBase(PluginManager::Manager<Trade::AbstractImporter>* manager, Float maxThreshold, Float meanThreshold): _maxThreshold{maxThreshold}, _meanThreshold{meanThreshold}, _max{}, _mean{} {
if(manager) _fileState.reset(new FileState{*manager});
CORRADE_ASSERT(!Math::isNan(maxThreshold) && !Math::isInf(maxThreshold) &&
!Math::isNan(meanThreshold) && !Math::isInf(meanThreshold),
"DebugTools::CompareImage: thresholds can't be NaN or infinity", );
CORRADE_ASSERT(meanThreshold <= maxThreshold,
"DebugTools::CompareImage: maxThreshold can't be smaller than meanThreshold", );
}
@ -359,12 +389,20 @@ bool ImageComparatorBase::operator()(const ImageView2D& actual, const ImageView2
Containers::Array<Float> delta;
std::tie(delta, _max, _mean) = DebugTools::Implementation::calculateImageDelta(actual, expected);
/* If both values are not above threshold, success */
if(_max > _maxThreshold && _mean > _meanThreshold)
/* Verify the max/mean is never below zero so we didn't mess up when
calculating specials. Note the inverted condition to catch NaNs in
_mean. The max should OTOH be never special as it would make all other
deltas become zero in comparison. */
CORRADE_INTERNAL_ASSERT(!(_mean < 0.0f));
CORRADE_INTERNAL_ASSERT(_max >= 0.0f && !Math::isInf(_max) && !Math::isNan(_max));
/* If both values are not above threshold, success. Comparing this way in
order to properly catch NaNs in mean values. */
if(_max > _maxThreshold && !(_mean <= _meanThreshold))
_state = State::AboveThresholds;
else if(_max > _maxThreshold)
_state = State::AboveMaxThreshold;
else if(_mean > _meanThreshold)
else if(!(_mean <= _meanThreshold))
_state = State::AboveMeanThreshold;
else return true;

24
src/Magnum/DebugTools/CompareImage.h

@ -206,6 +206,30 @@ printed as characters of different perceived brightness. Blocks with delta over
the max threshold are colored red, blocks with delta over the mean threshold
are colored yellow. The delta list contains X,Y pixel position (with origin at
bottom left), actual and expected pixel value and calculated delta.
@section DebugTools-CompareImage-specials Special floating-point values
For floating-point input, the comparator treats the values similarly to how
@ref Corrade::TestSuite::Comparator<float> behaves for scalars:
- If both actual and expected channel value are NaN, they are treated as the
same (with channel delta being 0).
- If actual and expected channel value have the same sign of infinity, they
are treated the same (with channel delta being 0).
- Otherwise, the delta is calculated the usual way, with NaN and infinity
values getting propagated according to floating-point rules. This means
the final per-pixel @f$ \Delta_{\boldsymbol{p}} @f$ becomes either NaN or
infinity.
- When calculating the max value, NaN and infinity @f$ \Delta_{\boldsymbol{p}} @f$
values are ignored. This is done in order to avoid a single infinity deltas
causing all other deltas to be comparatively zero in the ASCII-art
representation.
- The mean value is calculated as usual, meaning that NaN or infinity in
@f$ \Delta_{\boldsymbol{p}} @f$ "poison" the final value, reliably causing
the comparison to fail.
For the ASCII-art representation, NaN and infinity @f$ \Delta_{\boldsymbol{p}} @f$
values are always treated as maximum difference.
*/
class CompareImage {
public:

244
src/Magnum/DebugTools/Test/CompareImageTest.cpp

@ -53,13 +53,17 @@ struct CompareImageTest: TestSuite::Tester {
void calculateDelta();
void calculateDeltaStorage();
void calculateDeltaSpecials();
void calculateDeltaSpecials3();
void deltaImage();
void deltaImageScaling();
void deltaImageColors();
void deltaImageSpecials();
void pixelDelta();
void pixelDeltaOverflow();
void pixelDeltaSpecials();
void compareDifferentSize();
void compareDifferentFormat();
@ -67,6 +71,9 @@ struct CompareImageTest: TestSuite::Tester {
void compareAboveThresholds();
void compareAboveMaxThreshold();
void compareAboveMeanThreshold();
void compareSpecials();
void compareSpecialsMeanOnly();
void compareSpecialsDisallowedThreshold();
void setupExternalPluginManager();
void teardownExternalPluginManager();
@ -102,13 +109,17 @@ CompareImageTest::CompareImageTest() {
&CompareImageTest::calculateDelta,
&CompareImageTest::calculateDeltaStorage,
&CompareImageTest::calculateDeltaSpecials,
&CompareImageTest::calculateDeltaSpecials3,
&CompareImageTest::deltaImage,
&CompareImageTest::deltaImageScaling,
&CompareImageTest::deltaImageColors,
&CompareImageTest::deltaImageSpecials,
&CompareImageTest::pixelDelta,
&CompareImageTest::pixelDeltaOverflow,
&CompareImageTest::pixelDeltaSpecials,
&CompareImageTest::compareDifferentSize,
&CompareImageTest::compareDifferentFormat,
@ -116,6 +127,9 @@ CompareImageTest::CompareImageTest() {
&CompareImageTest::compareAboveThresholds,
&CompareImageTest::compareAboveMaxThreshold,
&CompareImageTest::compareAboveMeanThreshold,
&CompareImageTest::compareSpecials,
&CompareImageTest::compareSpecialsMeanOnly,
&CompareImageTest::compareSpecialsDisallowedThreshold,
&CompareImageTest::image,
&CompareImageTest::imageError});
@ -254,6 +268,72 @@ void CompareImageTest::calculateDeltaStorage() {
CORRADE_COMPARE(mean, 18.5f);
}
/* Variants:
- expected number, got inf (and inverse)
- expected number, got nan (and inverse)
- got inf in both (and again, but different sign)
- got nan in both
- got a number in both (twice, to ensure it's calculated correctly) */
const Float ActualDataSpecials[]{
Constants::inf(), 0.3f,
Constants::nan(), 0.3f,
-Constants::inf(), -Constants::inf(),
Constants::nan(),
0.3f, 3.0f
};
const Float ExpectedDataSpecials[]{
1.0f, -Constants::inf(),
0.3f, Constants::nan(),
-Constants::inf(), Constants::inf(),
Constants::nan(),
0.65f, -0.1f
};
const Float DeltaSpecials[]{
Constants::inf(), Constants::inf(),
Constants::nan(), Constants::nan(),
0.0f, Constants::inf(),
0.0f,
0.35f, 3.1f
};
const ImageView2D ActualSpecials{PixelFormat::R32F, {9, 1}, ActualDataSpecials};
const ImageView2D ExpectedSpecials{PixelFormat::R32F, {9, 1}, ExpectedDataSpecials};
void CompareImageTest::calculateDeltaSpecials() {
Containers::Array<Float> delta;
Float max, mean;
std::tie(delta, max, mean) = Implementation::calculateImageDelta(ActualSpecials, ExpectedSpecials);
CORRADE_COMPARE_AS(Containers::arrayView(delta),
Containers::arrayView(DeltaSpecials),
TestSuite::Compare::Container);
/* Max should be calculated *without* the specials because otherwise every
other potential difference will be zero compared to infinity. OTOH mean
needs to get "poisoned" by those in order to have *something* to fail
the test with. */
CORRADE_COMPARE(max, 3.1f);
CORRADE_COMPARE(mean, -Constants::nan());
}
void CompareImageTest::calculateDeltaSpecials3() {
/* Same as calculateDeltaSpecials(), but reinterpreting the data as
a three-component vector in order to test per-component handling of
specials */
const ImageView2D actualSpecials3{PixelFormat::RGB32F, {3, 1}, ActualDataSpecials};
const ImageView2D expectedSpecials3{PixelFormat::RGB32F, {3, 1}, ExpectedDataSpecials};
Containers::Array<Float> delta;
Float max, mean;
std::tie(delta, max, mean) = Implementation::calculateImageDelta(actualSpecials3, expectedSpecials3);
CORRADE_COMPARE_AS(delta, (Containers::Array<Float>{Containers::InPlaceInit, {
Constants::nan(), Constants::nan(), 1.15f
}}), TestSuite::Compare::Container);
/* Max and mean should be calculated *without* the specials because
otherwise every other potential difference will be zero compared to
infinity */
CORRADE_COMPARE(max, 1.15f);
CORRADE_COMPARE(mean, -Constants::nan());
}
void CompareImageTest::deltaImage() {
std::ostringstream out;
Debug d{&out, Debug::Flag::DisableColors};
@ -325,6 +405,25 @@ void CompareImageTest::deltaImageColors() {
" |: ,|\n");
}
void CompareImageTest::deltaImageSpecials() {
const Float inf = Constants::inf(), nan = Constants::nan();
/* Duplicate the rows as the delta image visualizer merges each two to
preserve ratio */
const Float delta[]{0.7f, inf, 2.5f,
0.7f, nan, 2.5f,
nan, inf, 0.0f,
nan, inf, 0.0f};
std::ostringstream out;
Debug dc{&out, Debug::Flag::DisableColors};
Implementation::printDeltaImage(dc, delta, {3, 4}, 3.0f, 0.0f, 0.0f);
/* Should show the max value for NaN and infs and the usual things
otherwise */
CORRADE_COMPARE(out.str(),
" |MM |\n"
" |~M8|\n");
}
void CompareImageTest::pixelDelta() {
{
Debug() << "Visual verification -- some lines should be yellow, some red:";
@ -356,6 +455,33 @@ void CompareImageTest::pixelDeltaOverflow() {
" [2,0] Vector(0.9), expected Vector(0.6) (Δ = 0.3)");
}
void CompareImageTest::pixelDeltaSpecials() {
std::ostringstream out;
Debug d{&out, Debug::Flag::DisableColors};
Implementation::printPixelDeltas(d, DeltaSpecials, ActualSpecials, ExpectedSpecials, 1.5f, 0.5f, 10);
/* MSVC prints -nan(ind) instead of ±nan. But only sometimes. */
#ifdef _MSC_VER
CORRADE_COMPARE(out.str(),
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(-nan(ind)) (Δ = -nan(ind))\n"
" [2,0] Vector(-nan(ind)), expected Vector(0.3) (Δ = -nan(ind))\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)");
#else
CORRADE_COMPARE(out.str(),
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n"
" [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)");
#endif
}
void CompareImageTest::compareDifferentSize() {
std::stringstream out;
@ -454,6 +580,124 @@ void CompareImageTest::compareAboveMeanThreshold() {
" [1,0] #5647ec, expected #5610ed (Δ = 18.6667)\n");
}
void CompareImageTest::compareSpecials() {
std::stringstream out;
{
TestSuite::Comparator<CompareImage> compare{1.5f, 0.5f};
CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials));
Debug d{&out, Debug::Flag::DisableColors};
compare.printErrorMessage(d, "a", "b");
}
/* Apple platforms, Android, Emscripten and MinGW32 don't print signed
NaNs. This is *not* a libc++ thing, tho -- libc++ on Linux prints signed NaNs. */
#if defined(CORRADE_TARGET_APPLE) || defined(CORRADE_TARGET_ANDROID) || defined(CORRADE_TARGET_EMSCRIPTEN) || defined(__MINGW32__)
CORRADE_COMPARE(out.str(),
"Images a and b have both max and mean delta above threshold, actual 3.1/nan but at most 1.5/0.5 expected. Delta image:\n"
" |MMMM M ,M|\n"
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n"
" [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n");
/* MSVC prints -nan(ind) instead of ±nan. But only sometimes. */
#elif defined(_MSC_VER)
CORRADE_COMPARE(out.str(),
"Images a and b have both max and mean delta above threshold, actual 3.1/-nan(ind) but at most 1.5/0.5 expected. Delta image:\n"
" |MMMM M ,M|\n"
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(-nan(ind)) (Δ = nan)\n"
" [2,0] Vector(-nan(ind)), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n");
/* Linux */
#else
CORRADE_COMPARE(out.str(),
"Images a and b have both max and mean delta above threshold, actual 3.1/-nan but at most 1.5/0.5 expected. Delta image:\n"
" |MMMM M ,M|\n"
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n"
" [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n");
#endif
}
void CompareImageTest::compareSpecialsMeanOnly() {
std::stringstream out;
{
TestSuite::Comparator<CompareImage> compare{15.0f, 0.5f};
CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials));
Debug d{&out, Debug::Flag::DisableColors};
compare.printErrorMessage(d, "a", "b");
}
/* Apple platforms, Android, Emscripten and MinGW32 don't print signed
NaNs. This is *not* a libc++ thing, tho -- libc++ on Linux prints signed NaNs. */
#if defined(CORRADE_TARGET_APPLE) || defined(CORRADE_TARGET_ANDROID) || defined(CORRADE_TARGET_EMSCRIPTEN) || defined(__MINGW32__)
CORRADE_COMPARE(out.str(),
"Images a and b have mean delta above threshold, actual nan but at most 0.5 expected. Max delta 3.1 is below threshold 15. Delta image:\n"
" |MMMM M ,M|\n"
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n"
" [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n");
/* MSVC prints -nan(ind) instead of ±nan. But only sometimes. */
#elif defined(CORRADE_TARGET_WINDOWS) && defined(_MSC_VER)
CORRADE_COMPARE(out.str(),
"Images a and b have mean delta above threshold, actual -nan(ind) but at most 0.5 expected. Max delta 3.1 is below threshold 15. Delta image:\n"
" |MMMM M ,M|\n"
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(-nan(ind)) (Δ = nan)\n"
" [2,0] Vector(-nan(ind)), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n");
/* Linux */
#else
CORRADE_COMPARE(out.str(),
"Images a and b have mean delta above threshold, actual -nan but at most 0.5 expected. Max delta 3.1 is below threshold 15. Delta image:\n"
" |MMMM M ,M|\n"
" Pixels above max/mean threshold:\n"
" [5,0] Vector(-inf), expected Vector(inf) (Δ = inf)\n"
" [3,0] Vector(0.3), expected Vector(nan) (Δ = nan)\n"
" [2,0] Vector(nan), expected Vector(0.3) (Δ = nan)\n"
" [1,0] Vector(0.3), expected Vector(-inf) (Δ = inf)\n"
" [0,0] Vector(inf), expected Vector(1) (Δ = inf)\n"
" [8,0] Vector(3), expected Vector(-0.1) (Δ = 3.1)\n");
#endif
}
void CompareImageTest::compareSpecialsDisallowedThreshold() {
std::stringstream out;
{
Error redirectError{&out};
TestSuite::Comparator<CompareImage> a{Constants::inf(), 0.3f};
TestSuite::Comparator<CompareImage> b{0.3f, Constants::nan()};
}
CORRADE_COMPARE(out.str(),
"DebugTools::CompareImage: thresholds can't be NaN or infinity\n"
"DebugTools::CompareImage: thresholds can't be NaN or infinity\n");
}
void CompareImageTest::setupExternalPluginManager() {
_manager.emplace("nonexistent");
/* Load the plugin directly from the build tree. Otherwise it's either

Loading…
Cancel
Save