From ed6cde95363f4f763ed19b83a354a15af21408cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Wed, 31 Jul 2019 16:24:23 +0200 Subject: [PATCH] DebugTools: support --save-diagnostic in CompareImage. --- doc/changelog.dox | 3 + src/Magnum/DebugTools/CompareImage.cpp | 162 +++++++-- src/Magnum/DebugTools/CompareImage.h | 93 +++-- src/Magnum/DebugTools/Test/CMakeLists.txt | 8 + .../DebugTools/Test/CompareImageTest.cpp | 344 ++++++++++++++---- src/Magnum/DebugTools/Test/configure.h.cmake | 1 + 6 files changed, 476 insertions(+), 135 deletions(-) diff --git a/doc/changelog.dox b/doc/changelog.dox index 7e3df1d37..5141ab8d0 100644 --- a/doc/changelog.dox +++ b/doc/changelog.dox @@ -69,6 +69,9 @@ See also: screenshots - @ref DebugTools::CompareImage and variants now properly handle floating-point specials such as NaN or @f$ \infty @f$ in compared data +- @ref DebugTools::CompareImageFile and @ref DebugTools::CompareImageToFile + now support the new @ref TestSuite-Tester-save-diagnostic "--save-diagnostic option", + making it possible to save images when a comparison fails @subsubsection changelog-latest-new-gl GL library diff --git a/src/Magnum/DebugTools/CompareImage.cpp b/src/Magnum/DebugTools/CompareImage.cpp index 5c699afa4..a26a359aa 100644 --- a/src/Magnum/DebugTools/CompareImage.cpp +++ b/src/Magnum/DebugTools/CompareImage.cpp @@ -30,12 +30,14 @@ #include #include #include +#include #include "Magnum/ImageView.h" #include "Magnum/PixelFormat.h" #include "Magnum/Math/Functions.h" #include "Magnum/Math/Color.h" #include "Magnum/Math/Algorithms/KahanSum.h" +#include "Magnum/Trade/AbstractImageConverter.h" #include "Magnum/Trade/AbstractImporter.h" #include "Magnum/Trade/ImageData.h" @@ -346,23 +348,43 @@ enum class ImageComparatorBase::State: UnsignedByte { class ImageComparatorBase::FileState { public: - explicit FileState(PluginManager::Manager& importerManager): importerManager{&importerManager} {} - - explicit FileState(): _privateImporterManager{Containers::InPlaceInit}, importerManager{&*_privateImporterManager} {} + explicit FileState(PluginManager::Manager* importerManager, PluginManager::Manager* converterManager): _importerManager{importerManager}, _converterManager{converterManager} {} + + explicit FileState() {} + + /* Lazy-create the importer / converter if those weren't passed from + the outside. The importer might not be used at all if we are + comparing two image data (but in that case the FileState won't be + created at all); the converter will get used only very rarely for + the --save-failed option. Treat both the same lazy way to keep the + code straightforward. */ + PluginManager::Manager& importerManager() { + if(!_importerManager) _importerManager = &_privateImporterManager.emplace(); + return *_importerManager; + } + PluginManager::Manager& converterManager() { + if(!_converterManager) _converterManager = &_privateConverterManager.emplace(); + return *_converterManager; + } private: Containers::Optional> _privateImporterManager; + Containers::Optional> _privateConverterManager; + PluginManager::Manager* _importerManager{}; + PluginManager::Manager* _converterManager{}; public: - PluginManager::Manager* importerManager; std::string actualFilename, expectedFilename; Containers::Optional actualImageData, expectedImageData; /** @todo could at least the views have a NoCreate constructor? */ Containers::Optional actualImage, expectedImage; }; -ImageComparatorBase::ImageComparatorBase(PluginManager::Manager* importerManager, Float maxThreshold, Float meanThreshold): _maxThreshold{maxThreshold}, _meanThreshold{meanThreshold}, _max{}, _mean{} { - if(importerManager) _fileState.reset(new FileState{*importerManager}); +ImageComparatorBase::ImageComparatorBase(PluginManager::Manager* importerManager, PluginManager::Manager* converterManager, Float maxThreshold, Float meanThreshold): _maxThreshold{maxThreshold}, _meanThreshold{meanThreshold}, _max{}, _mean{} { + /* Only instantiate the file state if there's something to save -- if we + are comparing two image data, it won't be used at all */ + if(importerManager || converterManager) + _fileState.reset(new FileState{importerManager, converterManager}); CORRADE_ASSERT(!Math::isNan(maxThreshold) && !Math::isInf(maxThreshold) && !Math::isNan(meanThreshold) && !Math::isInf(meanThreshold), @@ -373,18 +395,22 @@ ImageComparatorBase::ImageComparatorBase(PluginManager::Manager delta; @@ -405,101 +431,151 @@ bool ImageComparatorBase::operator()(const ImageView2D& actual, const ImageView2 _state = State::AboveMaxThreshold; else if(!(_mean <= _meanThreshold)) _state = State::AboveMeanThreshold; - else return true; + else return TestSuite::ComparisonStatusFlags{}; /* Otherwise save the deltas and fail */ _delta = std::move(delta); - return false; + return TestSuite::ComparisonStatusFlag::Failed; } -bool ImageComparatorBase::operator()(const std::string& actual, const std::string& expected) { +TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const std::string& actual, const std::string& expected) { if(!_fileState) _fileState.reset(new FileState); _fileState->actualFilename = actual; _fileState->expectedFilename = expected; Containers::Pointer importer; - if(!(importer = _fileState->importerManager->loadAndInstantiate("AnyImageImporter"))) { + /* Can't load importer plugin. While we *could* save diagnostic in this + case too, it would make no sense as it's a Schrödinger image at this + point -- we have no idea if it's the same or not until we open it. */ + if(!(importer = _fileState->importerManager().loadAndInstantiate("AnyImageImporter"))) { _state = State::PluginLoadFailed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; } + /* Same here. We can't open the image for some reason (file missing? broken + plugin?), so can't know if it's the same or not. */ if(!importer->openFile(actual) || !(_fileState->actualImageData = importer->image2D(0))) { _state = State::ActualImageLoadFailed; - return false; - } - - if(!importer->openFile(expected) || !(_fileState->expectedImageData = importer->image2D(0))) { - _state = State::ExpectedImageLoadFailed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; } + /* If the actual data are compressed, we won't be able to compare them + (and probably neither save them back due to format mismatches). Don't + provide diagnostic in that case. */ if(_fileState->actualImageData->isCompressed()) { _state = State::ActualImageIsCompressed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; + } + + /* At this point we already know we successfully opened the actual file, + so save also the view on its parsed contents to avoid it going out of + scope. We're saving through an image converter, not the original file, + see saveDiagnostic() for reasons why. */ + _fileState->actualImage.emplace(*_fileState->actualImageData); + + /* Save a reference to the actual image so saveDiagnostic() can reach the + data even if we fail before the final data comparison (which does this + as well) */ + _actualImage = &*_fileState->actualImage; + + /* If the expected file can't be opened, we should still be able to save + the actual as a diagnostic. This could get also used to generate ground + truth data on the first-ever test run. */ + if(!importer->openFile(expected) || !(_fileState->expectedImageData = importer->image2D(0))) { + _state = State::ExpectedImageLoadFailed; + return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic; } + /* If the expected file is compressed, it's bad, but it doesn't mean we + couldn't save the actual file either */ if(_fileState->expectedImageData->isCompressed()) { _state = State::ExpectedImageIsCompressed; - return false; + return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic; } - _fileState->actualImage.emplace(*_fileState->actualImageData); + /* Save also a view on the expected image data and proxy to the actual data + comparison. If comparison failed, offer to save a diagnostic. */ _fileState->expectedImage.emplace(*_fileState->expectedImageData); - return operator()(*_fileState->actualImage, *_fileState->expectedImage); + TestSuite::ComparisonStatusFlags flags = operator()(*_fileState->actualImage, *_fileState->expectedImage); + if(flags & TestSuite::ComparisonStatusFlag::Failed) + flags |= TestSuite::ComparisonStatusFlag::Diagnostic; + return flags; } -bool ImageComparatorBase::operator()(const ImageView2D& actual, const std::string& expected) { +TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const ImageView2D& actual, const std::string& expected) { if(!_fileState) _fileState.reset(new FileState); _fileState->expectedFilename = expected; Containers::Pointer importer; - if(!(importer = _fileState->importerManager->loadAndInstantiate("AnyImageImporter"))) { + /* Can't load importer plugin. While we *could* save diagnostic in this + case too, it would make no sense as it's a Schrödinger image at this + point -- we have no idea if it's the same or not until we open it. */ + if(!(importer = _fileState->importerManager().loadAndInstantiate("AnyImageImporter"))) { _state = State::PluginLoadFailed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; } + /* Save a reference to the actual image so saveDiagnostic() can reach the + data even if we fail before the final data comparison (which does this + as well) */ + _actualImage = &actual; + + /* If the expected file can't be opened, we should still be able to save + the actual as a diagnostic. This could get also used to generate ground + truth data on the first-ever test run. */ if(!importer->openFile(expected) || !(_fileState->expectedImageData = importer->image2D(0))) { _state = State::ExpectedImageLoadFailed; - return false; + return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic; } + /* If the expected file is compressed, it's bad, but it doesn't mean we + couldn't save the actual file either */ if(_fileState->expectedImageData->isCompressed()) { _state = State::ExpectedImageIsCompressed; - return false; + return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic; } + /* Save a view on the expected image data and proxy to the actual data + comparison. If comparison failed, offer to save a diagnostic. */ _fileState->expectedImage.emplace(*_fileState->expectedImageData); - return operator()(actual, *_fileState->expectedImage); + TestSuite::ComparisonStatusFlags flags = operator()(actual, *_fileState->expectedImage); + if(flags & TestSuite::ComparisonStatusFlag::Failed) + flags |= TestSuite::ComparisonStatusFlag::Diagnostic; + return flags; } -bool ImageComparatorBase::operator()(const std::string& actual, const ImageView2D& expected) { +TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const std::string& actual, const ImageView2D& expected) { if(!_fileState) _fileState.reset(new FileState); _fileState->actualFilename = actual; + /* Here we are comparing against a view, not a file, so we cannot save + diagnostic in any case as we don't have the expected filename. This + behavior is consistent with TestSuite::Compare::FileToString. */ + Containers::Pointer importer; - if(!(importer = _fileState->importerManager->loadAndInstantiate("AnyImageImporter"))) { + if(!(importer = _fileState->importerManager().loadAndInstantiate("AnyImageImporter"))) { _state = State::PluginLoadFailed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; } if(!importer->openFile(actual) || !(_fileState->actualImageData = importer->image2D(0))) { _state = State::ActualImageLoadFailed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; } if(_fileState->actualImageData->isCompressed()) { _state = State::ActualImageIsCompressed; - return false; + return TestSuite::ComparisonStatusFlag::Failed; } _fileState->actualImage.emplace(*_fileState->actualImageData); return operator()(*_fileState->actualImage, expected); } -void ImageComparatorBase::printErrorMessage(Debug& out, const std::string& actual, const std::string& expected) const { +void ImageComparatorBase::printMessage(TestSuite::ComparisonStatusFlags, Debug& out, const std::string& actual, const std::string& expected) const { if(_state == State::PluginLoadFailed) { out << "AnyImageImporter plugin could not be loaded."; return; @@ -552,4 +628,18 @@ void ImageComparatorBase::printErrorMessage(Debug& out, const std::string& actua } } +void ImageComparatorBase::saveDiagnostic(TestSuite::ComparisonStatusFlags, Utility::Debug& out, const std::string& path) { + CORRADE_INTERNAL_ASSERT(_fileState); + CORRADE_INTERNAL_ASSERT(_actualImage); + + const std::string filename = Utility::Directory::join(path, Utility::Directory::filename(_fileState->expectedFilename)); + + /* Export the data the base view/view comparator saved. Ignore failures, + we're in the middle of a fail anyway (and everything will print messages + to the output nevertheless). */ + Containers::Pointer converter = _fileState->converterManager().loadAndInstantiate("AnyImageConverter"); + if(converter && converter->exportToFile(*_actualImage, filename)) + out << "->" << filename; +} + }}} diff --git a/src/Magnum/DebugTools/CompareImage.h b/src/Magnum/DebugTools/CompareImage.h index f35d3cb48..8f3cb6638 100644 --- a/src/Magnum/DebugTools/CompareImage.h +++ b/src/Magnum/DebugTools/CompareImage.h @@ -60,21 +60,23 @@ namespace Implementation { class MAGNUM_DEBUGTOOLS_EXPORT ImageComparatorBase { public: - explicit ImageComparatorBase(PluginManager::Manager* importerManager, Float maxThreshold, Float meanThreshold); + explicit ImageComparatorBase(PluginManager::Manager* importerManager, PluginManager::Manager* converterManager, Float maxThreshold, Float meanThreshold); - /*implicit*/ ImageComparatorBase(): ImageComparatorBase{nullptr, 0.0f, 0.0f} {} + /*implicit*/ ImageComparatorBase(): ImageComparatorBase{nullptr, nullptr, 0.0f, 0.0f} {} ~ImageComparatorBase(); - bool operator()(const ImageView2D& actual, const ImageView2D& expected); + TestSuite::ComparisonStatusFlags operator()(const ImageView2D& actual, const ImageView2D& expected); - bool operator()(const std::string& actual, const std::string& expected); + TestSuite::ComparisonStatusFlags operator()(const std::string& actual, const std::string& expected); - bool operator()(const std::string& actual, const ImageView2D& expected); + TestSuite::ComparisonStatusFlags operator()(const std::string& actual, const ImageView2D& expected); - bool operator()(const ImageView2D& actual, const std::string& expected); + TestSuite::ComparisonStatusFlags operator()(const ImageView2D& actual, const std::string& expected); - void printErrorMessage(Debug& out, const std::string& actual, const std::string& expected) const; + void printMessage(TestSuite::ComparisonStatusFlags flags, Debug& out, const std::string& actual, const std::string& expected) const; + + void saveDiagnostic(TestSuite::ComparisonStatusFlags flags, Utility::Debug& out, const std::string& path); private: class MAGNUM_DEBUGTOOLS_LOCAL FileState; @@ -85,7 +87,7 @@ class MAGNUM_DEBUGTOOLS_EXPORT ImageComparatorBase { Float _maxThreshold, _meanThreshold; State _state{}; - const ImageView2D *_actualImage, *_expectedImage; + const ImageView2D *_actualImage{}, *_expectedImage{}; Float _max, _mean; Containers::Array _delta; }; @@ -99,44 +101,44 @@ namespace Corrade { namespace TestSuite { template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator: public Magnum::DebugTools::Implementation::ImageComparatorBase { public: - explicit Comparator(Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{nullptr, maxThreshold, meanThreshold} {} + explicit Comparator(Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{nullptr, nullptr, maxThreshold, meanThreshold} {} /*implicit*/ Comparator(): Comparator{0.0f, 0.0f} {} - bool operator()(const Magnum::ImageView2D& actual, const Magnum::ImageView2D& expected) { + ComparisonStatusFlags operator()(const Magnum::ImageView2D& actual, const Magnum::ImageView2D& expected) { return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected); } }; template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator: public Magnum::DebugTools::Implementation::ImageComparatorBase { public: - explicit Comparator(PluginManager::Manager* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, maxThreshold, meanThreshold} {} + explicit Comparator(PluginManager::Manager* importerManager, PluginManager::Manager* converterManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, converterManager, maxThreshold, meanThreshold} {} - /*implicit*/ Comparator(): Comparator{nullptr, 0.0f, 0.0f} {} + /*implicit*/ Comparator(): Comparator{nullptr, nullptr, 0.0f, 0.0f} {} - bool operator()(const std::string& actual, const std::string& expected) { + ComparisonStatusFlags operator()(const std::string& actual, const std::string& expected) { return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected); } }; template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator: public Magnum::DebugTools::Implementation::ImageComparatorBase { public: - explicit Comparator(PluginManager::Manager* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, maxThreshold, meanThreshold} {} + explicit Comparator(PluginManager::Manager* importerManager, PluginManager::Manager* converterManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, converterManager, maxThreshold, meanThreshold} {} - /*implicit*/ Comparator(): Comparator{nullptr, 0.0f, 0.0f} {} + /*implicit*/ Comparator(): Comparator{nullptr, nullptr, 0.0f, 0.0f} {} - bool operator()(const Magnum::ImageView2D& actual, const std::string& expected) { + ComparisonStatusFlags operator()(const Magnum::ImageView2D& actual, const std::string& expected) { return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected); } }; template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator: public Magnum::DebugTools::Implementation::ImageComparatorBase { public: - explicit Comparator(PluginManager::Manager* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, maxThreshold, meanThreshold} {} + explicit Comparator(PluginManager::Manager* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, nullptr, maxThreshold, meanThreshold} {} /*implicit*/ Comparator(): Comparator{nullptr, 0.0f, 0.0f} {} - bool operator()(const std::string& actual, const Magnum::ImageView2D& expected) { + ComparisonStatusFlags operator()(const std::string& actual, const Magnum::ImageView2D& expected) { return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected); } }; @@ -287,6 +289,20 @@ available: See also @ref CompareImageToFile and @ref CompareFileToImage for comparing in-memory images to image files and vice versa. + +@section DebugTools-CompareImageFile-save-diagnostic Saving files for failed comparisons + +The comparator supports the @ref TestSuite-Tester-save-diagnostic "--save-diagnostic option" +--- if the comparison fails, it saves actual image data to given directory with +a filename and format matching the expected file, using the +@ref Trade::AnyImageConverter "AnyImageConverter" plugin. For this to work, +both @ref Trade::AnyImageConverter "AnyImageConverter" and the concrete +converter plugin need to be available. You can use it to perform a manual data comparison with an external tool or for example to quickly update expected test +data --- point the option to the directory with expected test files and let the +test overwrite them with actual results. The @ref CompareImageToFile variant +supports the same; the @ref CompareImage / @ref CompareFileToImage variants +don't since the comparison is done against in-memory data and so producing a +file isn't that helpful as in the other two variants. */ class CompareImageFile { public: @@ -297,7 +313,7 @@ class CompareImageFile { * @param meanThreshold Mean threshold. If mean delta over all pixels * is above this value, the comparison fails */ - explicit CompareImageFile(Float maxThreshold, Float meanThreshold): _c{nullptr, maxThreshold, meanThreshold} {} + explicit CompareImageFile(Float maxThreshold, Float meanThreshold): _c{nullptr, nullptr, maxThreshold, meanThreshold} {} /** * @brief Construct with an explicit plugin manager instance @@ -307,7 +323,7 @@ class CompareImageFile { * @param meanThreshold Mean threshold. If mean delta over all * pixels is above this value, the comparison fails */ - explicit CompareImageFile(PluginManager::Manager& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, maxThreshold, meanThreshold} {} + explicit CompareImageFile(PluginManager::Manager& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, nullptr, maxThreshold, meanThreshold} {} /** * @brief Construct with an explicit plugin manager instance and implicit thresholds @@ -315,7 +331,24 @@ class CompareImageFile { * Equivalent to calling @ref CompareImageFile(PluginManager::Manager&, Float, Float) * with zero values. */ - explicit CompareImageFile(PluginManager::Manager& importerManager): _c{&importerManager, 0.0f, 0.0f} {} + explicit CompareImageFile(PluginManager::Manager& importerManager): _c{&importerManager, nullptr, 0.0f, 0.0f} {} + + /** + * @brief Construct with an explicit importer and converter plugin manager instance + * @param importerManager Image importer plugin manager instance + * @param converterManager Image converter plugin manager instance + * @param maxThreshold Max threshold. If any pixel has delta above + * this value, this comparison fails + * @param meanThreshold Mean threshold. If mean delta over all + * pixels is above this value, the comparison fails + * + * This variant is rarely usable outside of testing environment, as the + * @p converterManager is only ever used when saving diagnostic for + * failed * comparison when the `--save-diagnostic` + * @ref TestSuite-Tester-save-diagnostic "command-line option" is + * specified. + */ + explicit CompareImageFile(PluginManager::Manager& importerManager, PluginManager::Manager& converterManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, &converterManager, maxThreshold, meanThreshold} {} /** * @brief Construct with implicit thresholds @@ -323,7 +356,7 @@ class CompareImageFile { * Equivalent to calling @ref CompareImageFile(Float, Float) with zero * values. */ - explicit CompareImageFile(): _c{nullptr, 0.0f, 0.0f} {} + explicit CompareImageFile(): _c{nullptr, nullptr, 0.0f, 0.0f} {} #ifndef DOXYGEN_GENERATING_OUTPUT TestSuite::Comparator& comparator() { @@ -354,7 +387,7 @@ class CompareImageToFile { * See @ref CompareImageFile::CompareImageFile(Float, Float) for more * information. */ - explicit CompareImageToFile(Float maxThreshold, Float meanThreshold): _c{nullptr, maxThreshold, meanThreshold} {} + explicit CompareImageToFile(Float maxThreshold, Float meanThreshold): _c{nullptr, nullptr, maxThreshold, meanThreshold} {} /** * @brief Construct with an explicit plugin manager instance @@ -362,7 +395,7 @@ class CompareImageToFile { * See @ref CompareImageFile::CompareImageFile(PluginManager::Manager&, Float, Float) * for more information. */ - explicit CompareImageToFile(PluginManager::Manager& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, maxThreshold, meanThreshold} {} + explicit CompareImageToFile(PluginManager::Manager& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, nullptr, maxThreshold, meanThreshold} {} /** * @brief Construct with an explicit plugin manager instance and implicit thresholds @@ -370,7 +403,15 @@ class CompareImageToFile { * Equivalent to calling @ref CompareImageToFile(PluginManager::Manager&, Float, Float) * with zero values. */ - explicit CompareImageToFile(PluginManager::Manager& importerManager): _c{&importerManager, 0.0f, 0.0f} {} + explicit CompareImageToFile(PluginManager::Manager& importerManager): _c{&importerManager, nullptr, 0.0f, 0.0f} {} + + /** + * @brief Construct with an explicit importer and converter plugin manager instance + * + * See @ref CompareImageFile::CompareImageFile(PluginManager::Manager&, PluginManager::Manager&, Float, Float) + * for more information. + */ + explicit CompareImageToFile(PluginManager::Manager& importerManager, PluginManager::Manager& imageConverterManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, &imageConverterManager, maxThreshold, meanThreshold} {} /** * @brief Implicit constructor @@ -378,7 +419,7 @@ class CompareImageToFile { * Equivalent to calling @ref CompareImageToFile(Float, Float) with zero * values. */ - explicit CompareImageToFile(): _c{nullptr, 0.0f, 0.0f} {} + explicit CompareImageToFile(): _c{nullptr, nullptr, 0.0f, 0.0f} {} #ifndef DOXYGEN_GENERATING_OUTPUT TestSuite::Comparator& comparator() { diff --git a/src/Magnum/DebugTools/Test/CMakeLists.txt b/src/Magnum/DebugTools/Test/CMakeLists.txt index 02eaaee11..11af44ab2 100644 --- a/src/Magnum/DebugTools/Test/CMakeLists.txt +++ b/src/Magnum/DebugTools/Test/CMakeLists.txt @@ -30,9 +30,11 @@ if(WITH_TRADE) if(CORRADE_TARGET_EMSCRIPTEN OR CORRADE_TARGET_ANDROID) set(DEBUGTOOLS_TEST_DIR ".") set(SCREENSHOTTEST_SAVE_DIR "write") + set(COMPAREIMAGETEST_SAVE_DIR "write") else() set(DEBUGTOOLS_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SCREENSHOTTEST_SAVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/write) + set(COMPAREIMAGETEST_SAVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/write) endif() # CMake before 3.8 has broken $ expressions for iOS (see @@ -75,9 +77,15 @@ if(WITH_TRADE) target_include_directories(DebugToolsCompareImageTest PRIVATE $) else() target_include_directories(DebugToolsCompareImageTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + if(WITH_ANYIMAGECONVERTER) + target_link_libraries(DebugToolsCompareImageTest PRIVATE AnyImageConverter) + endif() if(WITH_ANYIMAGEIMPORTER) target_link_libraries(DebugToolsCompareImageTest PRIVATE AnyImageImporter) endif() + if(WITH_TGAIMAGECONVERTER) + target_link_libraries(DebugToolsCompareImageTest PRIVATE TgaImageConverter) + endif() if(WITH_TGAIMPORTER) target_link_libraries(DebugToolsCompareImageTest PRIVATE TgaImporter) endif() diff --git a/src/Magnum/DebugTools/Test/CompareImageTest.cpp b/src/Magnum/DebugTools/Test/CompareImageTest.cpp index dafb3e466..a56cc1c7f 100644 --- a/src/Magnum/DebugTools/Test/CompareImageTest.cpp +++ b/src/Magnum/DebugTools/Test/CompareImageTest.cpp @@ -29,8 +29,10 @@ #include #include #include +#include #include #include +#include #include #include "Magnum/ImageView.h" @@ -38,6 +40,7 @@ #include "Magnum/DebugTools/CompareImage.h" #include "Magnum/Math/Functions.h" #include "Magnum/Math/Color.h" +#include "Magnum/Trade/AbstractImageConverter.h" #include "Magnum/Trade/AbstractImporter.h" #include "configure.h" @@ -100,6 +103,7 @@ struct CompareImageTest: TestSuite::Tester { private: Containers::Optional> _importerManager; + Containers::Optional> _converterManager; }; CompareImageTest::CompareImageTest() { @@ -174,6 +178,9 @@ CompareImageTest::CompareImageTest() { &CompareImageTest::teardownExternalPluginManager); addTests({&CompareImageTest::fileToImageActualIsCompressed}); + + /* Plugin manager setup is not done here, but in the + setupExternalPluginManager() function */ } const Float ActualRedData[] = { @@ -489,10 +496,12 @@ void CompareImageTest::compareDifferentSize() { ImageView2D b{PixelFormat::RG8UI, {3, 5}, nullptr}; { - Error e(&out); TestSuite::Comparator compare{{}, {}}; - CORRADE_VERIFY(!compare(a, b)); - compare.printErrorMessage(e, "a", "b"); + TestSuite::ComparisonStatusFlags flags = compare(a, b); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); + Error e(&out); + compare.printMessage(flags, e, "a", "b"); } CORRADE_COMPARE(out.str(), "Images a and b have different size, actual Vector(3, 4) but Vector(3, 5) expected.\n"); @@ -505,10 +514,12 @@ void CompareImageTest::compareDifferentFormat() { ImageView2D b{PixelFormat::RGB32F, {3, 4}, nullptr}; { - Error e(&out); TestSuite::Comparator compare{{}, {}}; - CORRADE_VERIFY(!compare(a, b)); - compare.printErrorMessage(e, "a", "b"); + TestSuite::ComparisonStatusFlags flags = compare(a, b); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); + Error e(&out); + compare.printMessage(flags, e, "a", "b"); } CORRADE_COMPARE(out.str(), "Images a and b have different format, actual PixelFormat::RGBA32F but PixelFormat::RGB32F expected.\n"); @@ -523,7 +534,7 @@ void CompareImageTest::compareSameZeroThreshold() { }; const ImageView2D image{PixelFormat::RGB32F, {2, 2}, data}; - CORRADE_VERIFY((TestSuite::Comparator{0.0f, 0.0f}(image, image))); + CORRADE_COMPARE((TestSuite::Comparator{0.0f, 0.0f}(image, image)), TestSuite::ComparisonStatusFlags{}); } void CompareImageTest::compareAboveThresholds() { @@ -531,9 +542,11 @@ void CompareImageTest::compareAboveThresholds() { { TestSuite::Comparator compare{20.0f, 10.0f}; - CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb)); + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), @@ -550,9 +563,11 @@ void CompareImageTest::compareAboveMaxThreshold() { { TestSuite::Comparator compare{30.0f, 20.0f}; - CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb)); + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), @@ -567,9 +582,11 @@ void CompareImageTest::compareAboveMeanThreshold() { { TestSuite::Comparator compare{50.0f, 18.0f}; - CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb)); + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), @@ -585,9 +602,11 @@ void CompareImageTest::compareSpecials() { { TestSuite::Comparator compare{1.5f, 0.5f}; - CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials)); + TestSuite::ComparisonStatusFlags flags = compare(ActualSpecials, ExpectedSpecials); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } /* Apple platforms, Android, Emscripten and MinGW32 don't print signed @@ -637,9 +656,11 @@ void CompareImageTest::compareSpecialsMeanOnly() { { TestSuite::Comparator compare{15.0f, 0.5f}; - CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials)); + TestSuite::ComparisonStatusFlags flags = compare(ActualSpecials, ExpectedSpecials); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } /* Apple platforms, Android, Emscripten and MinGW32 don't print signed @@ -700,16 +721,26 @@ void CompareImageTest::compareSpecialsDisallowedThreshold() { void CompareImageTest::setupExternalPluginManager() { _importerManager.emplace("nonexistent"); + _converterManager.emplace("nonexistent"); /* Load the plugin directly from the build tree. Otherwise it's either static and already loaded or not present in the build tree */ - #if defined(ANYIMAGEIMPORTER_PLUGIN_FILENAME) && defined(TGAIMPORTER_PLUGIN_FILENAME) + #ifdef ANYIMAGEIMPORTER_PLUGIN_FILENAME CORRADE_INTERNAL_ASSERT(_importerManager->load(ANYIMAGEIMPORTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded); + #endif + #ifdef ANYIMAGECONVERTER_PLUGIN_FILENAME + CORRADE_INTERNAL_ASSERT(_converterManager->load(ANYIMAGECONVERTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded); + #endif + #ifdef TGAIMPORTER_PLUGIN_FILENAME CORRADE_INTERNAL_ASSERT(_importerManager->load(TGAIMPORTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded); #endif + #ifdef TGAIMAGECONVERTER_PLUGIN_FILENAME + CORRADE_INTERNAL_ASSERT(_converterManager->load(TGAIMAGECONVERTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded); + #endif } void CompareImageTest::teardownExternalPluginManager() { _importerManager = Containers::NullOpt; + _converterManager = Containers::NullOpt; } constexpr const char* ImageCompareError = @@ -722,6 +753,10 @@ constexpr const char* ImageCompareError = void CompareImageTest::image() { CORRADE_COMPARE_WITH(ActualRgb, ExpectedRgb, (CompareImage{40.0f, 20.0f})); + + /* No diagnostic as there's no error */ + TestSuite::Comparator compare{40.0f, 20.0f}; + CORRADE_COMPARE(compare(ActualRgb, ExpectedRgb), TestSuite::ComparisonStatusFlags{}); } void CompareImageTest::imageError() { @@ -729,9 +764,11 @@ void CompareImageTest::imageError() { { TestSuite::Comparator compare{20.0f, 10.0f}; - CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb)); + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), ImageCompareError); @@ -746,6 +783,13 @@ void CompareImageTest::imageFile() { Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"), (CompareImageFile{*_importerManager, 40.0f, 20.0f})); + + /* No diagnostic as there's no error */ + TestSuite::Comparator compare{&*_importerManager, nullptr, 40.0f, 20.0f}; + CORRADE_COMPARE(compare( + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")), + TestSuite::ComparisonStatusFlags{}); } void CompareImageTest::imageFileError() { @@ -755,16 +799,40 @@ void CompareImageTest::imageFileError() { std::stringstream out; + TestSuite::Comparator compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare( + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")); + /* The diagnostic flag should be slapped on the failure coming from the + operator() comparing two ImageViews */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic); + { - TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"))); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), ImageCompareError); + + /* Create the output dir if it doesn't exist, but avoid stale files making + false positives */ + CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR)); + std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageExpected.tga"); + if(Utility::Directory::exists(filename)) + CORRADE_VERIFY(Utility::Directory::rm(filename)); + + { + out.str({}); + Debug redirectOutput(&out); + compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR); + } + + /* We expect the *actual* contents, but under the *expected* filename. + Comparing file contents, expecting the converter makes exactly the same + file. */ + CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename)); + CORRADE_COMPARE_AS(filename, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File); } void CompareImageTest::imageFilePluginLoadFailed() { @@ -775,12 +843,15 @@ void CompareImageTest::imageFilePluginLoadFailed() { std::stringstream out; { - TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( + TestSuite::Comparator compare{&manager, nullptr, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare( Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"))); + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")); + /* Can't load a plugin, so we can't open the file, so we can't save + it either and thus no diagnostic */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "AnyImageImporter plugin could not be loaded.\n"); @@ -794,11 +865,14 @@ void CompareImageTest::imageFileActualLoadFailed() { std::stringstream out; { - TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare("nonexistent.tga", - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"))); + TestSuite::Comparator compare{&*_importerManager, nullptr, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare("nonexistent.tga", + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")); + /* We can't open the file, so we can't save it either and thus no + diagnostic */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "Actual image a (nonexistent.tga) could not be loaded.\n"); @@ -811,16 +885,39 @@ void CompareImageTest::imageFileExpectedLoadFailed() { std::stringstream out; + TestSuite::Comparator compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare( + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), + "nonexistent.tga"); + /* Actual file *could* be loaded, so save it! */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic); + { - TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), - "nonexistent.tga")); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "Expected image b (nonexistent.tga) could not be loaded.\n"); + + /* Create the output dir if it doesn't exist, but avoid stale files making + false positives */ + CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR)); + std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "nonexistent.tga"); + if(Utility::Directory::exists(filename)) + CORRADE_VERIFY(Utility::Directory::rm(filename)); + + { + out.str({}); + Debug redirectOutput(&out); + compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR); + } + + /* We expect the *actual* contents, but under the *expected* filename. + Comparing file contents, expecting the converter makes exactly the same + file. */ + CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename)); + CORRADE_COMPARE_AS(filename, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File); } void CompareImageTest::imageFileActualIsCompressed() { @@ -832,12 +929,15 @@ void CompareImageTest::imageFileActualIsCompressed() { std::stringstream out; { - TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( + TestSuite::Comparator compare{&manager, nullptr, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare( Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"), - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"))); + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")); + /* We most probably couldn't save the file because it's in a different + format, so no diagnostic */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."), "Actual image a (.../CompareImageCompressed.dds) is compressed, comparison not possible.\n"); @@ -851,17 +951,30 @@ void CompareImageTest::imageFileExpectedIsCompressed() { std::stringstream out; + TestSuite::Comparator compare{&manager, nullptr, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare( + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds")); + /* Actual file is not, so save it! */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic); + { - TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"))); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } - CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."), - "Expected image b (.../CompareImageCompressed.dds) is compressed, comparison not possible.\n"); + /* Create the output dir if it doesn't exist, but avoid stale files making + false positives */ + CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR)); + std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageCompressed.dds"); + if(Utility::Directory::exists(filename)) + CORRADE_VERIFY(Utility::Directory::rm(filename)); + + /* This will attempt to save a DDS and then fails, because we have no DDS + exporter. */ + Debug d; + compare.saveDiagnostic(flags, d, COMPAREIMAGETEST_SAVE_DIR); + CORRADE_VERIFY(!Utility::Directory::exists(filename)); } void CompareImageTest::imageToFile() { @@ -872,6 +985,11 @@ void CompareImageTest::imageToFile() { CORRADE_COMPARE_WITH(ActualRgb, Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"), (CompareImageToFile{*_importerManager, 40.0f, 20.0f})); + + /* No diagnostic as there's no error */ + TestSuite::Comparator compare{&*_importerManager, nullptr, 40.0f, 20.0f}; + CORRADE_COMPARE(compare(ActualRgb, Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")), + TestSuite::ComparisonStatusFlags{}); } void CompareImageTest::imageToFileError() { @@ -881,15 +999,39 @@ void CompareImageTest::imageToFileError() { std::stringstream out; + TestSuite::Comparator compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")); + /* The diagnostic flag should be slapped on the failure coming from the + operator() comparing two ImageViews */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic); + { - TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare(ActualRgb, - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"))); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), ImageCompareError); + + /* Create the output dir if it doesn't exist, but avoid stale files making + false positives */ + CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR)); + std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageExpected.tga"); + if(Utility::Directory::exists(filename)) + CORRADE_VERIFY(Utility::Directory::rm(filename)); + + { + out.str({}); + Debug redirectOutput(&out); + compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR); + } + + /* We expect the *actual* contents, but under the *expected* filename. + Comparing file contents, expecting the converter makes exactly the same + file. */ + CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename)); + CORRADE_COMPARE_AS(filename, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File); } void CompareImageTest::imageToFilePluginLoadFailed() { @@ -900,11 +1042,14 @@ void CompareImageTest::imageToFilePluginLoadFailed() { std::stringstream out; { - TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare(ActualRgb, - Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"))); + TestSuite::Comparator compare{&manager, nullptr, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")); + /* Can't load a plugin, so we can't open the file, so we can't save + it either and thus no diagnostic */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "AnyImageImporter plugin could not be loaded.\n"); @@ -917,14 +1062,37 @@ void CompareImageTest::imageToFileExpectedLoadFailed() { std::stringstream out; + TestSuite::Comparator compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, "nonexistent.tga"); + /* Actual file *could* be loaded, so save it! */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic); + { - TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare(ActualRgb, "nonexistent.tga")); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "Expected image b (nonexistent.tga) could not be loaded.\n"); + + /* Create the output dir if it doesn't exist, but avoid stale files making + false positives */ + CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR)); + std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "nonexistent.tga"); + if(Utility::Directory::exists(filename)) + CORRADE_VERIFY(Utility::Directory::rm(filename)); + + { + out.str({}); + Debug redirectOutput(&out); + compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR); + } + + /* We expect the *actual* contents, but under the *expected* filename. + Comparing file contents, expecting the converter makes exactly the same + file. */ + CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename)); + CORRADE_COMPARE_AS(filename, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File); } void CompareImageTest::imageToFileExpectedIsCompressed() { @@ -935,15 +1103,32 @@ void CompareImageTest::imageToFileExpectedIsCompressed() { std::stringstream out; + TestSuite::Comparator compare{&manager, nullptr, 20.0f, 10.0f}; + TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, + Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds")); + /* Actual file is not, so save it! */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic); + { - TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare(ActualRgb, Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"))); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."), "Expected image b (.../CompareImageCompressed.dds) is compressed, comparison not possible.\n"); + + /* Create the output dir if it doesn't exist, but avoid stale files making + false positives */ + CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR)); + std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageCompressed.dds"); + if(Utility::Directory::exists(filename)) + CORRADE_VERIFY(Utility::Directory::rm(filename)); + + /* This will attempt to save a DDS and then fails, because we have no DDS + exporter. */ + Debug d; + compare.saveDiagnostic(flags, d, COMPAREIMAGETEST_SAVE_DIR); + CORRADE_VERIFY(!Utility::Directory::exists(filename)); } void CompareImageTest::fileToImage() { @@ -955,6 +1140,11 @@ void CompareImageTest::fileToImage() { Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), ExpectedRgb, (CompareFileToImage{*_importerManager, 40.0f, 20.0f})); + + /* No diagnostic as there's no error */ + TestSuite::Comparator compare{&*_importerManager, 40.0f, 20.0f}; + CORRADE_COMPARE(compare(Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), + ExpectedRgb), TestSuite::ComparisonStatusFlags{}); } void CompareImageTest::fileToImageError() { @@ -966,11 +1156,13 @@ void CompareImageTest::fileToImageError() { { TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( + TestSuite::ComparisonStatusFlags flags = compare( Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), - ExpectedRgb)); + ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), ImageCompareError); @@ -985,11 +1177,13 @@ void CompareImageTest::fileToImagePluginLoadFailed() { { TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare( + TestSuite::ComparisonStatusFlags flags = compare( Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), - ExpectedRgb)); + ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "AnyImageImporter plugin could not be loaded.\n"); @@ -1004,9 +1198,11 @@ void CompareImageTest::fileToImageActualLoadFailed() { { TestSuite::Comparator compare{&*_importerManager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare("nonexistent.tga", ExpectedRgb)); + TestSuite::ComparisonStatusFlags flags = compare("nonexistent.tga", ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(out.str(), "Actual image a (nonexistent.tga) could not be loaded.\n"); @@ -1022,9 +1218,11 @@ void CompareImageTest::fileToImageActualIsCompressed() { { TestSuite::Comparator compare{&manager, 20.0f, 10.0f}; - CORRADE_VERIFY(!compare(Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"), ExpectedRgb)); + TestSuite::ComparisonStatusFlags flags = compare(Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"), ExpectedRgb); + /* No diagnostic as we don't have any expected filename */ + CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed); Debug d{&out, Debug::Flag::DisableColors}; - compare.printErrorMessage(d, "a", "b"); + compare.printMessage(flags, d, "a", "b"); } CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."), diff --git a/src/Magnum/DebugTools/Test/configure.h.cmake b/src/Magnum/DebugTools/Test/configure.h.cmake index c457c75ca..626e3471f 100644 --- a/src/Magnum/DebugTools/Test/configure.h.cmake +++ b/src/Magnum/DebugTools/Test/configure.h.cmake @@ -29,3 +29,4 @@ #cmakedefine TGAIMPORTER_PLUGIN_FILENAME "${TGAIMPORTER_PLUGIN_FILENAME}" #define DEBUGTOOLS_TEST_DIR "${DEBUGTOOLS_TEST_DIR}" #define SCREENSHOTTEST_SAVE_DIR "${SCREENSHOTTEST_SAVE_DIR}" +#define COMPAREIMAGETEST_SAVE_DIR "${COMPAREIMAGETEST_SAVE_DIR}"