DebugTools: support --save-diagnostic in CompareImage.

doc/changelog.dox

@@ -69,6 +69,9 @@ See also:
     screenshots
 -   @ref DebugTools::CompareImage and variants now properly handle
     floating-point specials such as NaN or @f$ \infty @f$ in compared data
+-   @ref DebugTools::CompareImageFile and @ref DebugTools::CompareImageToFile
+    now support the new @ref TestSuite-Tester-save-diagnostic "--save-diagnostic option",
+    making it possible to save images when a comparison fails
 
 @subsubsection changelog-latest-new-gl GL library
 

src/Magnum/DebugTools/CompareImage.cpp

@@ -30,12 +30,14 @@
 #include <Corrade/Containers/Optional.h>
 #include <Corrade/PluginManager/Manager.h>
 #include <Corrade/Utility/DebugStl.h>
+#include <Corrade/Utility/Directory.h>
 
 #include "Magnum/ImageView.h"
 #include "Magnum/PixelFormat.h"
 #include "Magnum/Math/Functions.h"
 #include "Magnum/Math/Color.h"
 #include "Magnum/Math/Algorithms/KahanSum.h"
+#include "Magnum/Trade/AbstractImageConverter.h"
 #include "Magnum/Trade/AbstractImporter.h"
 #include "Magnum/Trade/ImageData.h"
 
@@ -346,23 +348,43 @@ enum class ImageComparatorBase::State: UnsignedByte {
 
 class ImageComparatorBase::FileState {
     public:
-        explicit FileState(PluginManager::Manager<Trade::AbstractImporter>& importerManager): importerManager{&importerManager} {}
-
-        explicit FileState(): _privateImporterManager{Containers::InPlaceInit}, importerManager{&*_privateImporterManager} {}
+        explicit FileState(PluginManager::Manager<Trade::AbstractImporter>* importerManager, PluginManager::Manager<Trade::AbstractImageConverter>* converterManager): _importerManager{importerManager}, _converterManager{converterManager} {}
+
+        explicit FileState() {}
+
+        /* Lazy-create the importer / converter if those weren't passed from
+           the outside. The importer might not be used at all if we are
+           comparing two image data (but in that case the FileState won't be
+           created at all); the converter will get used only very rarely for
+           the --save-failed option. Treat both the same lazy way to keep the
+           code straightforward. */
+        PluginManager::Manager<Trade::AbstractImporter>& importerManager() {
+            if(!_importerManager) _importerManager = &_privateImporterManager.emplace();
+            return *_importerManager;
+        }
+        PluginManager::Manager<Trade::AbstractImageConverter>& converterManager() {
+            if(!_converterManager) _converterManager = &_privateConverterManager.emplace();
+            return *_converterManager;
+        }
 
     private:
         Containers::Optional<PluginManager::Manager<Trade::AbstractImporter>> _privateImporterManager;
+        Containers::Optional<PluginManager::Manager<Trade::AbstractImageConverter>> _privateConverterManager;
+        PluginManager::Manager<Trade::AbstractImporter>* _importerManager{};
+        PluginManager::Manager<Trade::AbstractImageConverter>* _converterManager{};
 
     public:
-        PluginManager::Manager<Trade::AbstractImporter>* importerManager;
         std::string actualFilename, expectedFilename;
         Containers::Optional<Trade::ImageData2D> actualImageData, expectedImageData;
         /** @todo could at least the views have a NoCreate constructor? */
         Containers::Optional<ImageView2D> actualImage, expectedImage;
 };
 
-ImageComparatorBase::ImageComparatorBase(PluginManager::Manager<Trade::AbstractImporter>* importerManager, Float maxThreshold, Float meanThreshold): _maxThreshold{maxThreshold}, _meanThreshold{meanThreshold}, _max{}, _mean{} {
-    if(importerManager) _fileState.reset(new FileState{*importerManager});
+ImageComparatorBase::ImageComparatorBase(PluginManager::Manager<Trade::AbstractImporter>* importerManager, PluginManager::Manager<Trade::AbstractImageConverter>* converterManager, Float maxThreshold, Float meanThreshold): _maxThreshold{maxThreshold}, _meanThreshold{meanThreshold}, _max{}, _mean{} {
+    /* Only instantiate the file state if there's something to save -- if we
+       are comparing two image data, it won't be used at all */
+    if(importerManager || converterManager)
+        _fileState.reset(new FileState{importerManager, converterManager});
 
     CORRADE_ASSERT(!Math::isNan(maxThreshold) && !Math::isInf(maxThreshold) &&
                    !Math::isNan(meanThreshold) && !Math::isInf(meanThreshold),
@@ -373,18 +395,22 @@ ImageComparatorBase::ImageComparatorBase(PluginManager::Manager<Trade::AbstractI
 
 ImageComparatorBase::~ImageComparatorBase() = default;
 
-bool ImageComparatorBase::operator()(const ImageView2D& actual, const ImageView2D& expected) {
+TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const ImageView2D& actual, const ImageView2D& expected) {
+    /* Taking just references is okay because in case of files those are
+       references to actual views stored inside FileState; and in case of
+       data, the actual/expected params passed here stay in scope for the whole
+       time where operator(), printMessage() & saveDiagnostic() gets called */
     _actualImage = &actual;
     _expectedImage = &expected;
 
     /* Verify that the images are the same */
     if(actual.size() != expected.size()) {
         _state = State::DifferentSize;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
     if(actual.format() != expected.format()) {
         _state = State::DifferentFormat;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
     Containers::Array<Float> delta;
@@ -405,101 +431,151 @@ bool ImageComparatorBase::operator()(const ImageView2D& actual, const ImageView2
         _state = State::AboveMaxThreshold;
     else if(!(_mean <= _meanThreshold))
         _state = State::AboveMeanThreshold;
-    else return true;
+    else return TestSuite::ComparisonStatusFlags{};
 
     /* Otherwise save the deltas and fail */
     _delta = std::move(delta);
-    return false;
+    return TestSuite::ComparisonStatusFlag::Failed;
 }
 
-bool ImageComparatorBase::operator()(const std::string& actual, const std::string& expected) {
+TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const std::string& actual, const std::string& expected) {
     if(!_fileState) _fileState.reset(new FileState);
 
     _fileState->actualFilename = actual;
     _fileState->expectedFilename = expected;
 
     Containers::Pointer<Trade::AbstractImporter> importer;
-    if(!(importer = _fileState->importerManager->loadAndInstantiate("AnyImageImporter"))) {
+    /* Can't load importer plugin. While we *could* save diagnostic in this
+       case too, it would make no sense as it's a Schrödinger image at this
+       point -- we have no idea if it's the same or not until we open it. */
+    if(!(importer = _fileState->importerManager().loadAndInstantiate("AnyImageImporter"))) {
         _state = State::PluginLoadFailed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
+    /* Same here. We can't open the image for some reason (file missing? broken
+       plugin?), so can't know if it's the same or not. */
     if(!importer->openFile(actual) || !(_fileState->actualImageData = importer->image2D(0))) {
         _state = State::ActualImageLoadFailed;
-        return false;
-    }
-
-    if(!importer->openFile(expected) || !(_fileState->expectedImageData = importer->image2D(0))) {
-        _state = State::ExpectedImageLoadFailed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
+    /* If the actual data are compressed, we won't be able to compare them
+       (and probably neither save them back due to format mismatches). Don't
+       provide diagnostic in that case. */
     if(_fileState->actualImageData->isCompressed()) {
         _state = State::ActualImageIsCompressed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
+    }
+
+    /* At this point we already know we successfully opened the actual file,
+       so save also the view on its parsed contents to avoid it going out of
+       scope. We're saving through an image converter, not the original file,
+       see saveDiagnostic() for reasons why. */
+    _fileState->actualImage.emplace(*_fileState->actualImageData);
+
+    /* Save a reference to the actual image so saveDiagnostic() can reach the
+       data even if we fail before the final data comparison (which does this
+       as well) */
+    _actualImage = &*_fileState->actualImage;
+
+    /* If the expected file can't be opened, we should still be able to save
+       the actual as a diagnostic. This could get also used to generate ground
+       truth data on the first-ever test run. */
+    if(!importer->openFile(expected) || !(_fileState->expectedImageData = importer->image2D(0))) {
+        _state = State::ExpectedImageLoadFailed;
+        return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic;
     }
 
+    /* If the expected file is compressed, it's bad, but it doesn't mean we
+       couldn't save the actual file either */
     if(_fileState->expectedImageData->isCompressed()) {
         _state = State::ExpectedImageIsCompressed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic;
     }
 
-    _fileState->actualImage.emplace(*_fileState->actualImageData);
+    /* Save also a view on the expected image data and proxy to the actual data
+       comparison. If comparison failed, offer to save a diagnostic. */
     _fileState->expectedImage.emplace(*_fileState->expectedImageData);
-    return operator()(*_fileState->actualImage, *_fileState->expectedImage);
+    TestSuite::ComparisonStatusFlags flags = operator()(*_fileState->actualImage, *_fileState->expectedImage);
+    if(flags & TestSuite::ComparisonStatusFlag::Failed)
+        flags |= TestSuite::ComparisonStatusFlag::Diagnostic;
+    return flags;
 }
 
-bool ImageComparatorBase::operator()(const ImageView2D& actual, const std::string& expected) {
+TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const ImageView2D& actual, const std::string& expected) {
     if(!_fileState) _fileState.reset(new FileState);
 
     _fileState->expectedFilename = expected;
 
     Containers::Pointer<Trade::AbstractImporter> importer;
-    if(!(importer = _fileState->importerManager->loadAndInstantiate("AnyImageImporter"))) {
+    /* Can't load importer plugin. While we *could* save diagnostic in this
+       case too, it would make no sense as it's a Schrödinger image at this
+       point -- we have no idea if it's the same or not until we open it. */
+    if(!(importer = _fileState->importerManager().loadAndInstantiate("AnyImageImporter"))) {
         _state = State::PluginLoadFailed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
+    /* Save a reference to the actual image so saveDiagnostic() can reach the
+       data even if we fail before the final data comparison (which does this
+       as well) */
+    _actualImage = &actual;
+
+    /* If the expected file can't be opened, we should still be able to save
+       the actual as a diagnostic. This could get also used to generate ground
+       truth data on the first-ever test run. */
     if(!importer->openFile(expected) || !(_fileState->expectedImageData = importer->image2D(0))) {
         _state = State::ExpectedImageLoadFailed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic;
     }
 
+    /* If the expected file is compressed, it's bad, but it doesn't mean we
+       couldn't save the actual file either */
     if(_fileState->expectedImageData->isCompressed()) {
         _state = State::ExpectedImageIsCompressed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic;
     }
 
+    /* Save a view on the expected image data and proxy to the actual data
+       comparison. If comparison failed, offer to save a diagnostic. */
     _fileState->expectedImage.emplace(*_fileState->expectedImageData);
-    return operator()(actual, *_fileState->expectedImage);
+    TestSuite::ComparisonStatusFlags flags = operator()(actual, *_fileState->expectedImage);
+    if(flags & TestSuite::ComparisonStatusFlag::Failed)
+        flags |= TestSuite::ComparisonStatusFlag::Diagnostic;
+    return flags;
 }
 
-bool ImageComparatorBase::operator()(const std::string& actual, const ImageView2D& expected) {
+TestSuite::ComparisonStatusFlags ImageComparatorBase::operator()(const std::string& actual, const ImageView2D& expected) {
     if(!_fileState) _fileState.reset(new FileState);
 
     _fileState->actualFilename = actual;
 
+    /* Here we are comparing against a view, not a file, so we cannot save
+       diagnostic in any case as we don't have the expected filename. This
+       behavior is consistent with TestSuite::Compare::FileToString. */
+
     Containers::Pointer<Trade::AbstractImporter> importer;
-    if(!(importer = _fileState->importerManager->loadAndInstantiate("AnyImageImporter"))) {
+    if(!(importer = _fileState->importerManager().loadAndInstantiate("AnyImageImporter"))) {
         _state = State::PluginLoadFailed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
     if(!importer->openFile(actual) || !(_fileState->actualImageData = importer->image2D(0))) {
         _state = State::ActualImageLoadFailed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
     if(_fileState->actualImageData->isCompressed()) {
         _state = State::ActualImageIsCompressed;
-        return false;
+        return TestSuite::ComparisonStatusFlag::Failed;
     }
 
     _fileState->actualImage.emplace(*_fileState->actualImageData);
     return operator()(*_fileState->actualImage, expected);
 }
 
-void ImageComparatorBase::printErrorMessage(Debug& out, const std::string& actual, const std::string& expected) const {
+void ImageComparatorBase::printMessage(TestSuite::ComparisonStatusFlags, Debug& out, const std::string& actual, const std::string& expected) const {
     if(_state == State::PluginLoadFailed) {
         out << "AnyImageImporter plugin could not be loaded.";
         return;
@@ -552,4 +628,18 @@ void ImageComparatorBase::printErrorMessage(Debug& out, const std::string& actua
     }
 }
 
+void ImageComparatorBase::saveDiagnostic(TestSuite::ComparisonStatusFlags, Utility::Debug& out, const std::string& path) {
+    CORRADE_INTERNAL_ASSERT(_fileState);
+    CORRADE_INTERNAL_ASSERT(_actualImage);
+
+    const std::string filename = Utility::Directory::join(path, Utility::Directory::filename(_fileState->expectedFilename));
+
+    /* Export the data the base view/view comparator saved. Ignore failures,
+       we're in the middle of a fail anyway (and everything will print messages
+       to the output nevertheless). */
+    Containers::Pointer<Trade::AbstractImageConverter> converter = _fileState->converterManager().loadAndInstantiate("AnyImageConverter");
+    if(converter && converter->exportToFile(*_actualImage, filename))
+        out << "->" << filename;
+}
+
 }}}

src/Magnum/DebugTools/CompareImage.h

@@ -60,21 +60,23 @@ namespace Implementation {
 
 class MAGNUM_DEBUGTOOLS_EXPORT ImageComparatorBase {
     public:
-        explicit ImageComparatorBase(PluginManager::Manager<Trade::AbstractImporter>* importerManager, Float maxThreshold, Float meanThreshold);
+        explicit ImageComparatorBase(PluginManager::Manager<Trade::AbstractImporter>* importerManager, PluginManager::Manager<Trade::AbstractImageConverter>* converterManager, Float maxThreshold, Float meanThreshold);
 
-        /*implicit*/ ImageComparatorBase(): ImageComparatorBase{nullptr, 0.0f, 0.0f} {}
+        /*implicit*/ ImageComparatorBase(): ImageComparatorBase{nullptr, nullptr, 0.0f, 0.0f} {}
 
         ~ImageComparatorBase();
 
-        bool operator()(const ImageView2D& actual, const ImageView2D& expected);
+        TestSuite::ComparisonStatusFlags operator()(const ImageView2D& actual, const ImageView2D& expected);
 
-        bool operator()(const std::string& actual, const std::string& expected);
+        TestSuite::ComparisonStatusFlags operator()(const std::string& actual, const std::string& expected);
 
-        bool operator()(const std::string& actual, const ImageView2D& expected);
+        TestSuite::ComparisonStatusFlags operator()(const std::string& actual, const ImageView2D& expected);
 
-        bool operator()(const ImageView2D& actual, const std::string& expected);
+        TestSuite::ComparisonStatusFlags operator()(const ImageView2D& actual, const std::string& expected);
 
-        void printErrorMessage(Debug& out, const std::string& actual, const std::string& expected) const;
+        void printMessage(TestSuite::ComparisonStatusFlags flags, Debug& out, const std::string& actual, const std::string& expected) const;
+
+        void saveDiagnostic(TestSuite::ComparisonStatusFlags flags, Utility::Debug& out, const std::string& path);
 
     private:
         class MAGNUM_DEBUGTOOLS_LOCAL FileState;
@@ -85,7 +87,7 @@ class MAGNUM_DEBUGTOOLS_EXPORT ImageComparatorBase {
         Float _maxThreshold, _meanThreshold;
 
         State _state{};
-        const ImageView2D *_actualImage, *_expectedImage;
+        const ImageView2D *_actualImage{}, *_expectedImage{};
         Float _max, _mean;
         Containers::Array<Float> _delta;
 };
@@ -99,44 +101,44 @@ namespace Corrade { namespace TestSuite {
 
 template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator<Magnum::DebugTools::CompareImage>: public Magnum::DebugTools::Implementation::ImageComparatorBase {
     public:
-        explicit Comparator(Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{nullptr, maxThreshold, meanThreshold} {}
+        explicit Comparator(Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{nullptr, nullptr, maxThreshold, meanThreshold} {}
 
         /*implicit*/ Comparator(): Comparator{0.0f, 0.0f} {}
 
-        bool operator()(const Magnum::ImageView2D& actual, const Magnum::ImageView2D& expected) {
+        ComparisonStatusFlags operator()(const Magnum::ImageView2D& actual, const Magnum::ImageView2D& expected) {
             return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected);
         }
 };
 
 template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator<Magnum::DebugTools::CompareImageFile>: public Magnum::DebugTools::Implementation::ImageComparatorBase {
     public:
-        explicit Comparator(PluginManager::Manager<Magnum::Trade::AbstractImporter>* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, maxThreshold, meanThreshold} {}
+        explicit Comparator(PluginManager::Manager<Magnum::Trade::AbstractImporter>* importerManager, PluginManager::Manager<Magnum::Trade::AbstractImageConverter>* converterManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, converterManager, maxThreshold, meanThreshold} {}
 
-        /*implicit*/ Comparator(): Comparator{nullptr, 0.0f, 0.0f} {}
+        /*implicit*/ Comparator(): Comparator{nullptr, nullptr, 0.0f, 0.0f} {}
 
-        bool operator()(const std::string& actual, const std::string& expected) {
+        ComparisonStatusFlags operator()(const std::string& actual, const std::string& expected) {
             return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected);
         }
 };
 
 template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator<Magnum::DebugTools::CompareImageToFile>: public Magnum::DebugTools::Implementation::ImageComparatorBase {
     public:
-        explicit Comparator(PluginManager::Manager<Magnum::Trade::AbstractImporter>* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, maxThreshold, meanThreshold} {}
+        explicit Comparator(PluginManager::Manager<Magnum::Trade::AbstractImporter>* importerManager, PluginManager::Manager<Magnum::Trade::AbstractImageConverter>* converterManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, converterManager, maxThreshold, meanThreshold} {}
 
-        /*implicit*/ Comparator(): Comparator{nullptr, 0.0f, 0.0f} {}
+        /*implicit*/ Comparator(): Comparator{nullptr, nullptr, 0.0f, 0.0f} {}
 
-        bool operator()(const Magnum::ImageView2D& actual, const std::string& expected) {
+        ComparisonStatusFlags operator()(const Magnum::ImageView2D& actual, const std::string& expected) {
             return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected);
         }
 };
 
 template<> class MAGNUM_DEBUGTOOLS_EXPORT Comparator<Magnum::DebugTools::CompareFileToImage>: public Magnum::DebugTools::Implementation::ImageComparatorBase {
     public:
-        explicit Comparator(PluginManager::Manager<Magnum::Trade::AbstractImporter>* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, maxThreshold, meanThreshold} {}
+        explicit Comparator(PluginManager::Manager<Magnum::Trade::AbstractImporter>* importerManager, Magnum::Float maxThreshold, Magnum::Float meanThreshold): Magnum::DebugTools::Implementation::ImageComparatorBase{importerManager, nullptr, maxThreshold, meanThreshold} {}
 
         /*implicit*/ Comparator(): Comparator{nullptr, 0.0f, 0.0f} {}
 
-        bool operator()(const std::string& actual, const Magnum::ImageView2D& expected) {
+        ComparisonStatusFlags operator()(const std::string& actual, const Magnum::ImageView2D& expected) {
             return Magnum::DebugTools::Implementation::ImageComparatorBase::operator()(actual, expected);
         }
 };
@@ -287,6 +289,20 @@ available:
 
 See also @ref CompareImageToFile and @ref CompareFileToImage for comparing
 in-memory images to image files and vice versa.
+
+@section DebugTools-CompareImageFile-save-diagnostic Saving files for failed comparisons
+
+The comparator supports the @ref TestSuite-Tester-save-diagnostic "--save-diagnostic option"
+--- if the comparison fails, it saves actual image data to given directory with
+a filename and format matching the expected file, using the
+@ref Trade::AnyImageConverter "AnyImageConverter" plugin. For this to work,
+both @ref Trade::AnyImageConverter "AnyImageConverter" and the concrete
+converter plugin need to be available. You can use it to perform a manual data comparison with an external tool or for example to quickly update expected test
+data --- point the option to the directory with expected test files and let the
+test overwrite them with actual results. The @ref CompareImageToFile variant
+supports the same; the @ref CompareImage / @ref CompareFileToImage variants
+don't since the comparison is done against in-memory data and so producing a
+file isn't that helpful as in the other two variants.
 */
 class CompareImageFile {
     public:
@@ -297,7 +313,7 @@ class CompareImageFile {
          * @param meanThreshold Mean threshold. If mean delta over all pixels
          *      is above this value, the comparison fails
          */
-        explicit CompareImageFile(Float maxThreshold, Float meanThreshold): _c{nullptr, maxThreshold, meanThreshold} {}
+        explicit CompareImageFile(Float maxThreshold, Float meanThreshold): _c{nullptr, nullptr, maxThreshold, meanThreshold} {}
 
         /**
          * @brief Construct with an explicit plugin manager instance
@@ -307,7 +323,7 @@ class CompareImageFile {
          * @param meanThreshold     Mean threshold. If mean delta over all
          *      pixels is above this value, the comparison fails
          */
-        explicit CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, maxThreshold, meanThreshold} {}
+        explicit CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, nullptr, maxThreshold, meanThreshold} {}
 
         /**
          * @brief Construct with an explicit plugin manager instance and implicit thresholds
@@ -315,7 +331,24 @@ class CompareImageFile {
          * Equivalent to calling @ref CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>&, Float, Float)
          * with zero values.
          */
-        explicit CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager): _c{&importerManager, 0.0f, 0.0f} {}
+        explicit CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager): _c{&importerManager, nullptr, 0.0f, 0.0f} {}
+
+        /**
+         * @brief Construct with an explicit importer and converter plugin manager instance
+         * @param importerManager   Image importer plugin manager instance
+         * @param converterManager  Image converter plugin manager instance
+         * @param maxThreshold      Max threshold. If any pixel has delta above
+         *      this value, this comparison fails
+         * @param meanThreshold     Mean threshold. If mean delta over all
+         *      pixels is above this value, the comparison fails
+         *
+         * This variant is rarely usable outside of testing environment, as the
+         * @p converterManager is only ever used when saving diagnostic for
+         * failed * comparison when the `--save-diagnostic`
+         * @ref TestSuite-Tester-save-diagnostic "command-line option" is
+         * specified.
+         */
+        explicit CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager, PluginManager::Manager<Trade::AbstractImageConverter>& converterManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, &converterManager, maxThreshold, meanThreshold} {}
 
         /**
          * @brief Construct with implicit thresholds
@@ -323,7 +356,7 @@ class CompareImageFile {
          * Equivalent to calling @ref CompareImageFile(Float, Float) with zero
          * values.
          */
-        explicit CompareImageFile(): _c{nullptr, 0.0f, 0.0f} {}
+        explicit CompareImageFile(): _c{nullptr, nullptr, 0.0f, 0.0f} {}
 
         #ifndef DOXYGEN_GENERATING_OUTPUT
         TestSuite::Comparator<CompareImageFile>& comparator() {
@@ -354,7 +387,7 @@ class CompareImageToFile {
          * See @ref CompareImageFile::CompareImageFile(Float, Float) for more
          * information.
          */
-        explicit CompareImageToFile(Float maxThreshold, Float meanThreshold): _c{nullptr, maxThreshold, meanThreshold} {}
+        explicit CompareImageToFile(Float maxThreshold, Float meanThreshold): _c{nullptr, nullptr, maxThreshold, meanThreshold} {}
 
         /**
          * @brief Construct with an explicit plugin manager instance
@@ -362,7 +395,7 @@ class CompareImageToFile {
          * See @ref CompareImageFile::CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>&, Float, Float)
          * for more information.
          */
-        explicit CompareImageToFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, maxThreshold, meanThreshold} {}
+        explicit CompareImageToFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, nullptr, maxThreshold, meanThreshold} {}
 
         /**
          * @brief Construct with an explicit plugin manager instance and implicit thresholds
@@ -370,7 +403,15 @@ class CompareImageToFile {
          * Equivalent to calling @ref CompareImageToFile(PluginManager::Manager<Trade::AbstractImporter>&, Float, Float)
          * with zero values.
          */
-        explicit CompareImageToFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager): _c{&importerManager, 0.0f, 0.0f} {}
+        explicit CompareImageToFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager): _c{&importerManager, nullptr, 0.0f, 0.0f} {}
+
+        /**
+         * @brief Construct with an explicit importer and converter plugin manager instance
+         *
+         * See @ref CompareImageFile::CompareImageFile(PluginManager::Manager<Trade::AbstractImporter>&, PluginManager::Manager<Trade::AbstractImageConverter>&, Float, Float)
+         * for more information.
+         */
+        explicit CompareImageToFile(PluginManager::Manager<Trade::AbstractImporter>& importerManager, PluginManager::Manager<Trade::AbstractImageConverter>& imageConverterManager, Float maxThreshold, Float meanThreshold): _c{&importerManager, &imageConverterManager, maxThreshold, meanThreshold} {}
 
         /**
          * @brief Implicit constructor
@@ -378,7 +419,7 @@ class CompareImageToFile {
          * Equivalent to calling @ref CompareImageToFile(Float, Float) with zero
          * values.
          */
-        explicit CompareImageToFile(): _c{nullptr, 0.0f, 0.0f} {}
+        explicit CompareImageToFile(): _c{nullptr, nullptr, 0.0f, 0.0f} {}
 
         #ifndef DOXYGEN_GENERATING_OUTPUT
         TestSuite::Comparator<CompareImageToFile>& comparator() {

src/Magnum/DebugTools/Test/CMakeLists.txt

@@ -30,9 +30,11 @@ if(WITH_TRADE)
     if(CORRADE_TARGET_EMSCRIPTEN OR CORRADE_TARGET_ANDROID)
         set(DEBUGTOOLS_TEST_DIR ".")
         set(SCREENSHOTTEST_SAVE_DIR "write")
+        set(COMPAREIMAGETEST_SAVE_DIR "write")
     else()
         set(DEBUGTOOLS_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR})
         set(SCREENSHOTTEST_SAVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/write)
+        set(COMPAREIMAGETEST_SAVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/write)
     endif()
 
     # CMake before 3.8 has broken $<TARGET_FILE*> expressions for iOS (see
@@ -75,9 +77,15 @@ if(WITH_TRADE)
         target_include_directories(DebugToolsCompareImageTest PRIVATE $<TARGET_FILE_DIR:DebugToolsCompareImageTest>)
     else()
         target_include_directories(DebugToolsCompareImageTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+        if(WITH_ANYIMAGECONVERTER)
+            target_link_libraries(DebugToolsCompareImageTest PRIVATE AnyImageConverter)
+        endif()
         if(WITH_ANYIMAGEIMPORTER)
             target_link_libraries(DebugToolsCompareImageTest PRIVATE AnyImageImporter)
         endif()
+        if(WITH_TGAIMAGECONVERTER)
+            target_link_libraries(DebugToolsCompareImageTest PRIVATE TgaImageConverter)
+        endif()
         if(WITH_TGAIMPORTER)
             target_link_libraries(DebugToolsCompareImageTest PRIVATE TgaImporter)
         endif()

src/Magnum/DebugTools/Test/CompareImageTest.cpp

@@ -29,8 +29,10 @@
 #include <Corrade/PluginManager/Manager.h>
 #include <Corrade/TestSuite/Tester.h>
 #include <Corrade/TestSuite/Compare/Container.h>
+#include <Corrade/TestSuite/Compare/File.h>
 #include <Corrade/Utility/DebugStl.h>
 #include <Corrade/Utility/Directory.h>
+#include <Corrade/Utility/FormatStl.h>
 #include <Corrade/Utility/String.h>
 
 #include "Magnum/ImageView.h"
@@ -38,6 +40,7 @@
 #include "Magnum/DebugTools/CompareImage.h"
 #include "Magnum/Math/Functions.h"
 #include "Magnum/Math/Color.h"
+#include "Magnum/Trade/AbstractImageConverter.h"
 #include "Magnum/Trade/AbstractImporter.h"
 
 #include "configure.h"
@@ -100,6 +103,7 @@ struct CompareImageTest: TestSuite::Tester {
 
     private:
         Containers::Optional<PluginManager::Manager<Trade::AbstractImporter>> _importerManager;
+        Containers::Optional<PluginManager::Manager<Trade::AbstractImageConverter>> _converterManager;
 };
 
 CompareImageTest::CompareImageTest() {
@@ -174,6 +178,9 @@ CompareImageTest::CompareImageTest() {
         &CompareImageTest::teardownExternalPluginManager);
 
     addTests({&CompareImageTest::fileToImageActualIsCompressed});
+
+    /* Plugin manager setup is not done here, but in the
+       setupExternalPluginManager() function */
 }
 
 const Float ActualRedData[] = {
@@ -489,10 +496,12 @@ void CompareImageTest::compareDifferentSize() {
     ImageView2D b{PixelFormat::RG8UI, {3, 5}, nullptr};
 
     {
-        Error e(&out);
         TestSuite::Comparator<CompareImage> compare{{}, {}};
-        CORRADE_VERIFY(!compare(a, b));
-        compare.printErrorMessage(e, "a", "b");
+        TestSuite::ComparisonStatusFlags flags = compare(a, b);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
+        Error e(&out);
+        compare.printMessage(flags, e, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "Images a and b have different size, actual Vector(3, 4) but Vector(3, 5) expected.\n");
@@ -505,10 +514,12 @@ void CompareImageTest::compareDifferentFormat() {
     ImageView2D b{PixelFormat::RGB32F, {3, 4}, nullptr};
 
     {
-        Error e(&out);
         TestSuite::Comparator<CompareImage> compare{{}, {}};
-        CORRADE_VERIFY(!compare(a, b));
-        compare.printErrorMessage(e, "a", "b");
+        TestSuite::ComparisonStatusFlags flags = compare(a, b);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
+        Error e(&out);
+        compare.printMessage(flags, e, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "Images a and b have different format, actual PixelFormat::RGBA32F but PixelFormat::RGB32F expected.\n");
@@ -523,7 +534,7 @@ void CompareImageTest::compareSameZeroThreshold() {
     };
 
     const ImageView2D image{PixelFormat::RGB32F, {2, 2}, data};
-    CORRADE_VERIFY((TestSuite::Comparator<CompareImage>{0.0f, 0.0f}(image, image)));
+    CORRADE_COMPARE((TestSuite::Comparator<CompareImage>{0.0f, 0.0f}(image, image)), TestSuite::ComparisonStatusFlags{});
 }
 
 void CompareImageTest::compareAboveThresholds() {
@@ -531,9 +542,11 @@ void CompareImageTest::compareAboveThresholds() {
 
     {
         TestSuite::Comparator<CompareImage> compare{20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb));
+        TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(),
@@ -550,9 +563,11 @@ void CompareImageTest::compareAboveMaxThreshold() {
 
     {
         TestSuite::Comparator<CompareImage> compare{30.0f, 20.0f};
-        CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb));
+        TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(),
@@ -567,9 +582,11 @@ void CompareImageTest::compareAboveMeanThreshold() {
 
     {
         TestSuite::Comparator<CompareImage> compare{50.0f, 18.0f};
-        CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb));
+        TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(),
@@ -585,9 +602,11 @@ void CompareImageTest::compareSpecials() {
 
     {
         TestSuite::Comparator<CompareImage> compare{1.5f, 0.5f};
-        CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials));
+        TestSuite::ComparisonStatusFlags flags = compare(ActualSpecials, ExpectedSpecials);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     /* Apple platforms, Android, Emscripten and MinGW32 don't print signed
@@ -637,9 +656,11 @@ void CompareImageTest::compareSpecialsMeanOnly() {
 
     {
         TestSuite::Comparator<CompareImage> compare{15.0f, 0.5f};
-        CORRADE_VERIFY(!compare(ActualSpecials, ExpectedSpecials));
+        TestSuite::ComparisonStatusFlags flags = compare(ActualSpecials, ExpectedSpecials);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     /* Apple platforms, Android, Emscripten and MinGW32 don't print signed
@@ -700,16 +721,26 @@ void CompareImageTest::compareSpecialsDisallowedThreshold() {
 
 void CompareImageTest::setupExternalPluginManager() {
     _importerManager.emplace("nonexistent");
+    _converterManager.emplace("nonexistent");
     /* Load the plugin directly from the build tree. Otherwise it's either
        static and already loaded or not present in the build tree */
-    #if defined(ANYIMAGEIMPORTER_PLUGIN_FILENAME) && defined(TGAIMPORTER_PLUGIN_FILENAME)
+    #ifdef ANYIMAGEIMPORTER_PLUGIN_FILENAME
     CORRADE_INTERNAL_ASSERT(_importerManager->load(ANYIMAGEIMPORTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded);
+    #endif
+    #ifdef ANYIMAGECONVERTER_PLUGIN_FILENAME
+    CORRADE_INTERNAL_ASSERT(_converterManager->load(ANYIMAGECONVERTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded);
+    #endif
+    #ifdef TGAIMPORTER_PLUGIN_FILENAME
     CORRADE_INTERNAL_ASSERT(_importerManager->load(TGAIMPORTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded);
     #endif
+    #ifdef TGAIMAGECONVERTER_PLUGIN_FILENAME
+    CORRADE_INTERNAL_ASSERT(_converterManager->load(TGAIMAGECONVERTER_PLUGIN_FILENAME) & PluginManager::LoadState::Loaded);
+    #endif
 }
 
 void CompareImageTest::teardownExternalPluginManager() {
     _importerManager = Containers::NullOpt;
+    _converterManager = Containers::NullOpt;
 }
 
 constexpr const char* ImageCompareError =
@@ -722,6 +753,10 @@ constexpr const char* ImageCompareError =
 
 void CompareImageTest::image() {
     CORRADE_COMPARE_WITH(ActualRgb, ExpectedRgb, (CompareImage{40.0f, 20.0f}));
+
+    /* No diagnostic as there's no error */
+    TestSuite::Comparator<CompareImage> compare{40.0f, 20.0f};
+    CORRADE_COMPARE(compare(ActualRgb, ExpectedRgb), TestSuite::ComparisonStatusFlags{});
 }
 
 void CompareImageTest::imageError() {
@@ -729,9 +764,11 @@ void CompareImageTest::imageError() {
 
     {
         TestSuite::Comparator<CompareImage> compare{20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(ActualRgb, ExpectedRgb));
+        TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), ImageCompareError);
@@ -746,6 +783,13 @@ void CompareImageTest::imageFile() {
         Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
         Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"),
         (CompareImageFile{*_importerManager, 40.0f, 20.0f}));
+
+    /* No diagnostic as there's no error */
+    TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, nullptr, 40.0f, 20.0f};
+    CORRADE_COMPARE(compare(
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")),
+        TestSuite::ComparisonStatusFlags{});
 }
 
 void CompareImageTest::imageFileError() {
@@ -755,16 +799,40 @@ void CompareImageTest::imageFileError() {
 
     std::stringstream out;
 
+    TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f};
+    TestSuite::ComparisonStatusFlags flags = compare(
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"));
+    /* The diagnostic flag should be slapped on the failure coming from the
+       operator() comparing two ImageViews */
+    CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic);
+
     {
-        TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")));
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), ImageCompareError);
+
+    /* Create the output dir if it doesn't exist, but avoid stale files making
+       false positives */
+    CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR));
+    std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageExpected.tga");
+    if(Utility::Directory::exists(filename))
+        CORRADE_VERIFY(Utility::Directory::rm(filename));
+
+    {
+        out.str({});
+        Debug redirectOutput(&out);
+        compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR);
+    }
+
+    /* We expect the *actual* contents, but under the *expected* filename.
+       Comparing file contents, expecting the converter makes exactly the same
+       file. */
+    CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename));
+    CORRADE_COMPARE_AS(filename,
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File);
 }
 
 void CompareImageTest::imageFilePluginLoadFailed() {
@@ -775,12 +843,15 @@ void CompareImageTest::imageFilePluginLoadFailed() {
     std::stringstream out;
 
     {
-        TestSuite::Comparator<CompareImageFile> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
+        TestSuite::Comparator<CompareImageFile> compare{&manager, nullptr, 20.0f, 10.0f};
+        TestSuite::ComparisonStatusFlags flags = compare(
             Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")));
+            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"));
+        /* Can't load a plugin, so we can't open the file, so we can't save
+           it either and thus no diagnostic */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "AnyImageImporter plugin could not be loaded.\n");
@@ -794,11 +865,14 @@ void CompareImageTest::imageFileActualLoadFailed() {
     std::stringstream out;
 
     {
-        TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare("nonexistent.tga",
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")));
+        TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, nullptr, 20.0f, 10.0f};
+        TestSuite::ComparisonStatusFlags flags = compare("nonexistent.tga",
+            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"));
+        /* We can't open the file, so we can't save it either and thus no
+           diagnostic */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "Actual image a (nonexistent.tga) could not be loaded.\n");
@@ -811,16 +885,39 @@ void CompareImageTest::imageFileExpectedLoadFailed() {
 
     std::stringstream out;
 
+    TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f};
+    TestSuite::ComparisonStatusFlags flags = compare(
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
+        "nonexistent.tga");
+    /* Actual file *could* be loaded, so save it! */
+    CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic);
+
     {
-        TestSuite::Comparator<CompareImageFile> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
-            "nonexistent.tga"));
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "Expected image b (nonexistent.tga) could not be loaded.\n");
+
+    /* Create the output dir if it doesn't exist, but avoid stale files making
+       false positives */
+    CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR));
+    std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "nonexistent.tga");
+    if(Utility::Directory::exists(filename))
+        CORRADE_VERIFY(Utility::Directory::rm(filename));
+
+    {
+        out.str({});
+        Debug redirectOutput(&out);
+        compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR);
+    }
+
+    /* We expect the *actual* contents, but under the *expected* filename.
+       Comparing file contents, expecting the converter makes exactly the same
+       file. */
+    CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename));
+    CORRADE_COMPARE_AS(filename,
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File);
 }
 
 void CompareImageTest::imageFileActualIsCompressed() {
@@ -832,12 +929,15 @@ void CompareImageTest::imageFileActualIsCompressed() {
     std::stringstream out;
 
     {
-        TestSuite::Comparator<CompareImageFile> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
+        TestSuite::Comparator<CompareImageFile> compare{&manager, nullptr, 20.0f, 10.0f};
+        TestSuite::ComparisonStatusFlags flags = compare(
             Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"),
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")));
+            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"));
+        /* We most probably couldn't save the file because it's in a different
+           format, so no diagnostic */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."), "Actual image a (.../CompareImageCompressed.dds) is compressed, comparison not possible.\n");
@@ -851,17 +951,30 @@ void CompareImageTest::imageFileExpectedIsCompressed() {
 
     std::stringstream out;
 
+    TestSuite::Comparator<CompareImageFile> compare{&manager, nullptr, 20.0f, 10.0f};
+    TestSuite::ComparisonStatusFlags flags = compare(
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"));
+    /* Actual file is not, so save it! */
+    CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic);
+
     {
-        TestSuite::Comparator<CompareImageFile> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds")));
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
-    CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."),
-        "Expected image b (.../CompareImageCompressed.dds) is compressed, comparison not possible.\n");
+    /* Create the output dir if it doesn't exist, but avoid stale files making
+       false positives */
+    CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR));
+    std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageCompressed.dds");
+    if(Utility::Directory::exists(filename))
+        CORRADE_VERIFY(Utility::Directory::rm(filename));
+
+    /* This will attempt to save a DDS and then fails, because we have no DDS
+       exporter. */
+    Debug d;
+    compare.saveDiagnostic(flags, d, COMPAREIMAGETEST_SAVE_DIR);
+    CORRADE_VERIFY(!Utility::Directory::exists(filename));
 }
 
 void CompareImageTest::imageToFile() {
@@ -872,6 +985,11 @@ void CompareImageTest::imageToFile() {
     CORRADE_COMPARE_WITH(ActualRgb,
         Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"),
         (CompareImageToFile{*_importerManager, 40.0f, 20.0f}));
+
+    /* No diagnostic as there's no error */
+    TestSuite::Comparator<CompareImageToFile> compare{&*_importerManager, nullptr, 40.0f, 20.0f};
+    CORRADE_COMPARE(compare(ActualRgb, Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")),
+        TestSuite::ComparisonStatusFlags{});
 }
 
 void CompareImageTest::imageToFileError() {
@@ -881,15 +999,39 @@ void CompareImageTest::imageToFileError() {
 
     std::stringstream out;
 
+    TestSuite::Comparator<CompareImageToFile> compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f};
+    TestSuite::ComparisonStatusFlags flags = compare(ActualRgb,
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"));
+    /* The diagnostic flag should be slapped on the failure coming from the
+       operator() comparing two ImageViews */
+    CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic);
+
     {
-        TestSuite::Comparator<CompareImageToFile> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(ActualRgb,
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")));
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), ImageCompareError);
+
+    /* Create the output dir if it doesn't exist, but avoid stale files making
+       false positives */
+    CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR));
+    std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageExpected.tga");
+    if(Utility::Directory::exists(filename))
+        CORRADE_VERIFY(Utility::Directory::rm(filename));
+
+    {
+        out.str({});
+        Debug redirectOutput(&out);
+        compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR);
+    }
+
+    /* We expect the *actual* contents, but under the *expected* filename.
+       Comparing file contents, expecting the converter makes exactly the same
+       file. */
+    CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename));
+    CORRADE_COMPARE_AS(filename,
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File);
 }
 
 void CompareImageTest::imageToFilePluginLoadFailed() {
@@ -900,11 +1042,14 @@ void CompareImageTest::imageToFilePluginLoadFailed() {
     std::stringstream out;
 
     {
-        TestSuite::Comparator<CompareImageToFile> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(ActualRgb,
-            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga")));
+        TestSuite::Comparator<CompareImageToFile> compare{&manager, nullptr, 20.0f, 10.0f};
+        TestSuite::ComparisonStatusFlags flags = compare(ActualRgb,
+            Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageExpected.tga"));
+        /* Can't load a plugin, so we can't open the file, so we can't save
+           it either and thus no diagnostic */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "AnyImageImporter plugin could not be loaded.\n");
@@ -917,14 +1062,37 @@ void CompareImageTest::imageToFileExpectedLoadFailed() {
 
     std::stringstream out;
 
+    TestSuite::Comparator<CompareImageToFile> compare{&*_importerManager, &*_converterManager, 20.0f, 10.0f};
+    TestSuite::ComparisonStatusFlags flags = compare(ActualRgb, "nonexistent.tga");
+    /* Actual file *could* be loaded, so save it! */
+    CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic);
+
     {
-        TestSuite::Comparator<CompareImageToFile> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(ActualRgb, "nonexistent.tga"));
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "Expected image b (nonexistent.tga) could not be loaded.\n");
+
+    /* Create the output dir if it doesn't exist, but avoid stale files making
+       false positives */
+    CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR));
+    std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "nonexistent.tga");
+    if(Utility::Directory::exists(filename))
+        CORRADE_VERIFY(Utility::Directory::rm(filename));
+
+    {
+        out.str({});
+        Debug redirectOutput(&out);
+        compare.saveDiagnostic(flags, redirectOutput, COMPAREIMAGETEST_SAVE_DIR);
+    }
+
+    /* We expect the *actual* contents, but under the *expected* filename.
+       Comparing file contents, expecting the converter makes exactly the same
+       file. */
+    CORRADE_COMPARE(out.str(), Utility::formatString("-> {}\n", filename));
+    CORRADE_COMPARE_AS(filename,
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"), TestSuite::Compare::File);
 }
 
 void CompareImageTest::imageToFileExpectedIsCompressed() {
@@ -935,15 +1103,32 @@ void CompareImageTest::imageToFileExpectedIsCompressed() {
 
     std::stringstream out;
 
+    TestSuite::Comparator<CompareImageToFile> compare{&manager, nullptr, 20.0f, 10.0f};
+    TestSuite::ComparisonStatusFlags flags = compare(ActualRgb,
+        Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"));
+    /* Actual file is not, so save it! */
+    CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed|TestSuite::ComparisonStatusFlag::Diagnostic);
+
     {
-        TestSuite::Comparator<CompareImageToFile> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(ActualRgb, Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds")));
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."),
         "Expected image b (.../CompareImageCompressed.dds) is compressed, comparison not possible.\n");
+
+    /* Create the output dir if it doesn't exist, but avoid stale files making
+       false positives */
+    CORRADE_VERIFY(Utility::Directory::mkpath(COMPAREIMAGETEST_SAVE_DIR));
+    std::string filename = Utility::Directory::join(COMPAREIMAGETEST_SAVE_DIR, "CompareImageCompressed.dds");
+    if(Utility::Directory::exists(filename))
+        CORRADE_VERIFY(Utility::Directory::rm(filename));
+
+    /* This will attempt to save a DDS and then fails, because we have no DDS
+       exporter. */
+    Debug d;
+    compare.saveDiagnostic(flags, d, COMPAREIMAGETEST_SAVE_DIR);
+    CORRADE_VERIFY(!Utility::Directory::exists(filename));
 }
 
 void CompareImageTest::fileToImage() {
@@ -955,6 +1140,11 @@ void CompareImageTest::fileToImage() {
         Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
         ExpectedRgb,
         (CompareFileToImage{*_importerManager, 40.0f, 20.0f}));
+
+    /* No diagnostic as there's no error */
+    TestSuite::Comparator<CompareFileToImage> compare{&*_importerManager, 40.0f, 20.0f};
+    CORRADE_COMPARE(compare(Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
+        ExpectedRgb), TestSuite::ComparisonStatusFlags{});
 }
 
 void CompareImageTest::fileToImageError() {
@@ -966,11 +1156,13 @@ void CompareImageTest::fileToImageError() {
 
     {
         TestSuite::Comparator<CompareFileToImage> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
+        TestSuite::ComparisonStatusFlags flags = compare(
             Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
-            ExpectedRgb));
+            ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), ImageCompareError);
@@ -985,11 +1177,13 @@ void CompareImageTest::fileToImagePluginLoadFailed() {
 
     {
         TestSuite::Comparator<CompareFileToImage> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(
+        TestSuite::ComparisonStatusFlags flags = compare(
             Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageActual.tga"),
-            ExpectedRgb));
+            ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "AnyImageImporter plugin could not be loaded.\n");
@@ -1004,9 +1198,11 @@ void CompareImageTest::fileToImageActualLoadFailed() {
 
     {
         TestSuite::Comparator<CompareFileToImage> compare{&*_importerManager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare("nonexistent.tga", ExpectedRgb));
+        TestSuite::ComparisonStatusFlags flags = compare("nonexistent.tga", ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(out.str(), "Actual image a (nonexistent.tga) could not be loaded.\n");
@@ -1022,9 +1218,11 @@ void CompareImageTest::fileToImageActualIsCompressed() {
 
     {
         TestSuite::Comparator<CompareFileToImage> compare{&manager, 20.0f, 10.0f};
-        CORRADE_VERIFY(!compare(Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"), ExpectedRgb));
+        TestSuite::ComparisonStatusFlags flags = compare(Utility::Directory::join(DEBUGTOOLS_TEST_DIR, "CompareImageCompressed.dds"), ExpectedRgb);
+        /* No diagnostic as we don't have any expected filename */
+        CORRADE_COMPARE(flags, TestSuite::ComparisonStatusFlag::Failed);
         Debug d{&out, Debug::Flag::DisableColors};
-        compare.printErrorMessage(d, "a", "b");
+        compare.printMessage(flags, d, "a", "b");
     }
 
     CORRADE_COMPARE(Utility::String::replaceFirst(out.str(), DEBUGTOOLS_TEST_DIR, "..."),

src/Magnum/DebugTools/Test/configure.h.cmake

@@ -29,3 +29,4 @@
 #cmakedefine TGAIMPORTER_PLUGIN_FILENAME "${TGAIMPORTER_PLUGIN_FILENAME}"
 #define DEBUGTOOLS_TEST_DIR "${DEBUGTOOLS_TEST_DIR}"
 #define SCREENSHOTTEST_SAVE_DIR "${SCREENSHOTTEST_SAVE_DIR}"
+#define COMPAREIMAGETEST_SAVE_DIR "${COMPAREIMAGETEST_SAVE_DIR}"