Shaders: classical alpha masking support in Phong and Flat.

Slow and ugly, is here only for making quick'n'dirty alpha masked
drawing without a need for blending or depth sorting. Oh and also to
support the glTF alpha mask feature. Again, beware: *slow*.

doc/changelog.dox

@@ -90,6 +90,8 @@ See also:
 -   New dedicated @ref Shaders::VertexColor::Color3 and
     @ref Shaders::VertexColor::Color4 attribute specifiers for more convenient
     distinction between three- and four-component vertex color attribute.
+-   Classical alpha masking support in @ref Shaders::Flat and
+    @ref Shaders::Phong
 
 @subsubsection changelog-latest-new-trade Trade library
 

src/Magnum/Shaders/Flat.cpp

@@ -65,6 +65,7 @@ template<UnsignedInt dimensions> Flat<dimensions>::Flat(const Flags flags): _fla
         .addSource(rs.get("generic.glsl"))
         .addSource(rs.get(vertexShaderName<dimensions>()));
     frag.addSource(flags & Flag::Textured ? "#define TEXTURED\n" : "")
+        .addSource(flags & Flag::AlphaMask ? "#define ALPHA_MASK\n" : "")
         .addSource(rs.get("Flat.frag"));
 
     CORRADE_INTERNAL_ASSERT_OUTPUT(GL::Shader::compile({vert, frag}));
@@ -89,6 +90,7 @@ template<UnsignedInt dimensions> Flat<dimensions>::Flat(const Flags flags): _fla
     {
         _transformationProjectionMatrixUniform = uniformLocation("transformationProjectionMatrix");
         _colorUniform = uniformLocation("color");
+        if(flags & Flag::AlphaMask) _alphaMaskUniform = uniformLocation("alphaMask");
     }
 
     #ifndef MAGNUM_TARGET_GLES
@@ -102,6 +104,7 @@ template<UnsignedInt dimensions> Flat<dimensions>::Flat(const Flags flags): _fla
     #ifdef MAGNUM_TARGET_GLES
     /* Default to fully opaque white so we can see the texture */
     if(flags & Flag::Textured) setColor(Color4(1.0f));
+    if(flags & Flag::AlphaMask) setAlphaMask(0.5f);
     #endif
 }
 
@@ -112,6 +115,13 @@ template<UnsignedInt dimensions> Flat<dimensions>& Flat<dimensions>::bindTexture
     return *this;
 }
 
+template<UnsignedInt dimensions> Flat<dimensions>& Flat<dimensions>::setAlphaMask(Float mask) {
+    CORRADE_ASSERT(_flags & Flag::AlphaMask,
+        "Shaders::Flat::setAlphaMask(): the shader was not created with alpha mask enabled", *this);
+    setUniform(_alphaMaskUniform, mask);
+    return *this;
+}
+
 template class Flat<2>;
 template class Flat<3>;
 

src/Magnum/Shaders/Flat.frag

@@ -45,6 +45,17 @@ uniform lowp vec4 color
     #endif
     ;
 
+#ifdef ALPHA_MASK
+#ifdef EXPLICIT_UNIFORM_LOCATION
+layout(location = 2)
+#endif
+uniform lowp float alphaMask
+    #ifndef GL_ES
+    = 0.5
+    #endif
+    ;
+#endif
+
 #ifdef TEXTURED
 in mediump vec2 interpolatedTextureCoordinates;
 #endif
@@ -59,4 +70,8 @@ void main() {
         texture(textureData, interpolatedTextureCoordinates)*
         #endif
         color;
+
+    #ifdef ALPHA_MASK
+    if(fragmentColor.a < alphaMask) discard;
+    #endif
 }

src/Magnum/Shaders/Flat.h

@@ -40,7 +40,10 @@
 namespace Magnum { namespace Shaders {
 
 namespace Implementation {
-    enum class FlatFlag: UnsignedByte { Textured = 1 << 0 };
+    enum class FlatFlag: UnsignedByte {
+        Textured = 1 << 0,
+        AlphaMask = 1 << 1
+    };
     typedef Containers::EnumSet<FlatFlag> FlatFlags;
 }
 
@@ -83,6 +86,15 @@ Common rendering setup:
 
 @snippet MagnumShaders.cpp Flat-usage-textured2
 
+@subsection Shaders-Flat-usage-alpha Alpha blending and masking
+
+Enable @ref Flag::AlphaMask and tune @ref setAlphaMask() for simple
+binary alpha-masked drawing that doesn't require depth sorting or blending
+enabled. Note that this feature is implemented using the GLSL @glsl discard @ce
+operation which is known to have considerable performance impact on some
+platforms. With proper depth sorting and blending you'll usually get much
+better performance and output quality.
+
 @see @ref shaders, @ref Flat2D, @ref Flat3D
 */
 template<UnsignedInt dimensions> class MAGNUM_SHADERS_EXPORT Flat: public GL::AbstractShaderProgram {
@@ -112,7 +124,24 @@ template<UnsignedInt dimensions> class MAGNUM_SHADERS_EXPORT Flat: public GL::Ab
          * @see @ref Flags, @ref flags()
          */
         enum class Flag: UnsignedByte {
-            Textured = 1 << 0   /**< The shader uses texture instead of color */
+            /**
+             * Multiply color with a texture.
+             * @see @ref setColor(), @ref setTexture()
+             */
+            Textured = 1 << 0,
+
+            /**
+             * Enable alpha masking. If the combined fragment color has an
+             * alpha less than the value specified with @ref setAlphaMask(),
+             * given fragment is discarded.
+             *
+             * This uses the @glsl discard @ce operation which is known to have
+             * considerable performance impact on some platforms. While useful
+             * for cheap alpha masking that doesn't require depth sorting,
+             * with proper depth sorting and blending you'll usually get much
+             * better performance and output quality.
+             */
+            AlphaMask = 1 << 1
         };
 
         /**
@@ -192,6 +221,17 @@ template<UnsignedInt dimensions> class MAGNUM_SHADERS_EXPORT Flat: public GL::Ab
          */
         Flat<dimensions>& bindTexture(GL::Texture2D& texture);
 
+        /**
+         * @brief Set alpha mask value
+         * @return Reference to self (for method chaining)
+         *
+         * Expects that the shader was created with @ref Flag::AlphaMask
+         * enabled. Fragments with alpha values smaller than the mask value
+         * will be discarded. Default is @cpp 0.5f @ce. See the flag
+         * documentation for further information.
+         */
+        Flat<dimensions>& setAlphaMask(Float mask);
+
         #ifdef MAGNUM_BUILD_DEPRECATED
         /** @brief @copybrief bindTexture()
          * @deprecated Use @ref bindTexture() instead.
@@ -204,7 +244,8 @@ template<UnsignedInt dimensions> class MAGNUM_SHADERS_EXPORT Flat: public GL::Ab
     private:
         Flags _flags;
         Int _transformationProjectionMatrixUniform{0},
-            _colorUniform{1};
+            _colorUniform{1},
+            _alphaMaskUniform{2};
 };
 
 /** @brief 2D flat shader */

src/Magnum/Shaders/Phong.cpp

@@ -67,6 +67,7 @@ Phong::Phong(const Flags flags): _flags(flags) {
     frag.addSource(flags & Flag::AmbientTexture ? "#define AMBIENT_TEXTURE\n" : "")
         .addSource(flags & Flag::DiffuseTexture ? "#define DIFFUSE_TEXTURE\n" : "")
         .addSource(flags & Flag::SpecularTexture ? "#define SPECULAR_TEXTURE\n" : "")
+        .addSource(flags & Flag::AlphaMask ? "#define ALPHA_MASK\n" : "")
         .addSource(rs.get("Phong.frag"));
 
     CORRADE_INTERNAL_ASSERT_OUTPUT(GL::Shader::compile({vert, frag}));
@@ -81,7 +82,8 @@ Phong::Phong(const Flags flags): _flags(flags) {
     {
         bindAttributeLocation(Position::Location, "position");
         bindAttributeLocation(Normal::Location, "normal");
-        if(flags) bindAttributeLocation(TextureCoordinates::Location, "textureCoordinates");
+        if(flags & (Flag::AmbientTexture|Flag::DiffuseTexture|Flag::SpecularTexture))
+            bindAttributeLocation(TextureCoordinates::Location, "textureCoordinates");
     }
 
     CORRADE_INTERNAL_ASSERT_OUTPUT(link());
@@ -99,6 +101,7 @@ Phong::Phong(const Flags flags): _flags(flags) {
         _specularColorUniform = uniformLocation("specularColor");
         _lightColorUniform = uniformLocation("lightColor");
         _shininessUniform = uniformLocation("shininess");
+        if(flags & Flag::AlphaMask) _alphaMaskUniform = uniformLocation("alphaMask");
     }
 
     #ifndef MAGNUM_TARGET_GLES
@@ -121,6 +124,7 @@ Phong::Phong(const Flags flags): _flags(flags) {
     setSpecularColor(Color4{1.0f});
     setLightColor(Color4{1.0f});
     setShininess(80.0f);
+    if(flags & Flag::AlphaMask) setAlphaMask(0.5f);
     #endif
 }
 
@@ -152,4 +156,11 @@ Phong& Phong::bindTextures(GL::Texture2D* ambient, GL::Texture2D* diffuse, GL::T
     return *this;
 }
 
+Phong& Phong::setAlphaMask(Float mask) {
+    CORRADE_ASSERT(_flags & Flag::AlphaMask,
+        "Shaders::Phong::setAlphaMask(): the shader was not created with alpha mask enabled", *this);
+    setUniform(_alphaMaskUniform, mask);
+    return *this;
+}
+
 }}

src/Magnum/Shaders/Phong.frag

@@ -103,6 +103,17 @@ uniform mediump float shininess
     #endif
     ;
 
+#ifdef ALPHA_MASK
+#ifdef EXPLICIT_UNIFORM_LOCATION
+layout(location = 9)
+#endif
+uniform lowp float alphaMask
+    #ifndef GL_ES
+    = 0.5
+    #endif
+    ;
+#endif
+
 in mediump vec3 transformedNormal;
 in highp vec3 lightDirection;
 in highp vec3 cameraDirection;
@@ -148,4 +159,8 @@ void main() {
         mediump float specularity = pow(max(0.0, dot(normalize(cameraDirection), reflection)), shininess);
         color += finalSpecularColor*specularity;
     }
+
+    #ifdef ALPHA_MASK
+    if(color.a < alphaMask) discard;
+    #endif
 }

src/Magnum/Shaders/Phong.h

@@ -76,7 +76,14 @@ Common rendering setup:
 
 @snippet MagnumShaders.cpp Phong-usage-texture2
 
-@subsection Shaders-Phong-usage-alpha Alpha-masked drawing
+@subsection Shaders-Phong-usage-alpha Alpha blending and masking
+
+Enable @ref Flag::AlphaMask and tune @ref setAlphaMask() for simple
+binary alpha-masked drawing that doesn't require depth sorting or blending
+enabled. Note that this feature is implemented using the GLSL @glsl discard @ce
+operation which is known to have considerable performance impact on some
+platforms. With proper depth sorting and blending you'll usually get much
+better performance and output quality.
 
 For general alpha-masked drawing you need to provide ambient texture with alpha
 channel and set alpha channel of diffuse/specular color to `0.0f` so only
@@ -138,7 +145,20 @@ class MAGNUM_SHADERS_EXPORT Phong: public GL::AbstractShaderProgram {
              * Multiply specular color with a texture.
              * @see @ref setSpecularColor(), @ref setSpecularTexture()
              */
-            SpecularTexture = 1 << 2
+            SpecularTexture = 1 << 2,
+
+            /**
+             * Enable alpha masking. If the combined fragment color has an
+             * alpha less than the value specified with @ref setAlphaMask(),
+             * given fragment is discarded.
+             *
+             * This uses the @glsl discard @ce operation which is known to have
+             * considerable performance impact on some platforms. While useful
+             * for cheap alpha masking that doesn't require depth sorting,
+             * with proper depth sorting and blending you'll usually get much
+             * better performance and output quality.
+             */
+            AlphaMask = 1 << 3
         };
 
         /**
@@ -317,6 +337,17 @@ class MAGNUM_SHADERS_EXPORT Phong: public GL::AbstractShaderProgram {
             return *this;
         }
 
+        /**
+         * @brief Set alpha mask value
+         * @return Reference to self (for method chaining)
+         *
+         * Expects that the shader was created with @ref Flag::AlphaMask
+         * enabled. Fragments with alpha values smaller than the mask value
+         * will be discarded. Default is @cpp 0.5f @ce. See the flag
+         * documentation for further information.
+         */
+        Phong& setAlphaMask(Float mask);
+
         /**
          * @brief Set transformation matrix
          * @return Reference to self (for method chaining)
@@ -377,7 +408,8 @@ class MAGNUM_SHADERS_EXPORT Phong: public GL::AbstractShaderProgram {
             _diffuseColorUniform{5},
             _specularColorUniform{6},
             _lightColorUniform{7},
-            _shininessUniform{8};
+            _shininessUniform{8},
+            _alphaMaskUniform{9};
 };
 
 CORRADE_ENUMSET_OPERATORS(Phong::Flags)

src/Magnum/Shaders/Test/FlatGLTest.cpp

@@ -44,6 +44,9 @@ struct FlatGLTest: GL::OpenGLTester {
 
     template<UnsignedInt dimensions> void bindTexture();
     template<UnsignedInt dimensions> void bindTextureNotEnabled();
+
+    template<UnsignedInt dimensions> void setAlphaMask();
+    template<UnsignedInt dimensions> void setAlphaMaskNotEnabled();
 };
 
 namespace {
@@ -71,7 +74,12 @@ FlatGLTest::FlatGLTest() {
         &FlatGLTest::bindTexture<2>,
         &FlatGLTest::bindTexture<3>,
         &FlatGLTest::bindTextureNotEnabled<2>,
-        &FlatGLTest::bindTextureNotEnabled<3>});
+        &FlatGLTest::bindTextureNotEnabled<3>,
+
+        &FlatGLTest::setAlphaMask<2>,
+        &FlatGLTest::setAlphaMask<3>,
+        &FlatGLTest::setAlphaMaskNotEnabled<2>,
+        &FlatGLTest::setAlphaMaskNotEnabled<3>});
 }
 
 template<UnsignedInt dimensions> void FlatGLTest::construct() {
@@ -143,6 +151,25 @@ template<UnsignedInt dimensions> void FlatGLTest::bindTextureNotEnabled() {
     CORRADE_COMPARE(out.str(), "Shaders::Flat::bindTexture(): the shader was not created with texturing enabled\n");
 }
 
+template<UnsignedInt dimensions> void FlatGLTest::setAlphaMask() {
+    /* Test just that no assertion is fired */
+    Flat<dimensions> shader{Flat<dimensions>::Flag::AlphaMask};
+    shader.setAlphaMask(0.25f);
+
+    MAGNUM_VERIFY_NO_GL_ERROR();
+}
+
+template<UnsignedInt dimensions> void FlatGLTest::setAlphaMaskNotEnabled() {
+    std::ostringstream out;
+    Error redirectError{&out};
+
+    Flat<dimensions> shader;
+    shader.setAlphaMask(0.75f);
+
+    CORRADE_COMPARE(out.str(),
+        "Shaders::Flat::setAlphaMask(): the shader was not created with alpha mask enabled\n");
+}
+
 }}}
 
 CORRADE_TEST_MAIN(Magnum::Shaders::Test::FlatGLTest)

src/Magnum/Shaders/Test/PhongGLTest.cpp

@@ -43,6 +43,9 @@ struct PhongGLTest: GL::OpenGLTester {
 
     void bindTextures();
     void bindTexturesNotEnabled();
+
+    void setAlphaMask();
+    void setAlphaMaskNotEnabled();
 };
 
 constexpr struct {
@@ -56,7 +59,9 @@ constexpr struct {
     {"ambient + diffuse texture", Phong::Flag::AmbientTexture|Phong::Flag::DiffuseTexture},
     {"ambient + specular texture", Phong::Flag::AmbientTexture|Phong::Flag::SpecularTexture},
     {"diffuse + specular texture", Phong::Flag::DiffuseTexture|Phong::Flag::SpecularTexture},
-    {"ambient + diffuse + specular texture", Phong::Flag::AmbientTexture|Phong::Flag::DiffuseTexture|Phong::Flag::SpecularTexture}};
+    {"ambient + diffuse + specular texture", Phong::Flag::AmbientTexture|Phong::Flag::DiffuseTexture|Phong::Flag::SpecularTexture},
+    {"alpha mask", Phong::Flag::AlphaMask},
+    {"alpha mask + diffuse texture", Phong::Flag::AlphaMask|Phong::Flag::DiffuseTexture}
 };
 
 PhongGLTest::PhongGLTest() {
@@ -65,7 +70,10 @@ PhongGLTest::PhongGLTest() {
     addTests({&PhongGLTest::constructMove,
 
               &PhongGLTest::bindTextures,
-              &PhongGLTest::bindTexturesNotEnabled});
+              &PhongGLTest::bindTexturesNotEnabled,
+
+              &PhongGLTest::setAlphaMask,
+              &PhongGLTest::setAlphaMaskNotEnabled});
 }
 
 void PhongGLTest::construct() {
@@ -139,6 +147,25 @@ void PhongGLTest::bindTexturesNotEnabled() {
         "Shaders::Phong::bindTextures(): the shader was not created with any textures enabled\n");
 }
 
+void PhongGLTest::setAlphaMask() {
+    /* Test just that no assertion is fired */
+    Phong shader{Phong::Flag::AlphaMask};
+    shader.setAlphaMask(0.25f);
+
+    MAGNUM_VERIFY_NO_GL_ERROR();
+}
+
+void PhongGLTest::setAlphaMaskNotEnabled() {
+    std::ostringstream out;
+    Error redirectError{&out};
+
+    Phong shader;
+    shader.setAlphaMask(0.75f);
+
+    CORRADE_COMPARE(out.str(),
+        "Shaders::Phong::setAlphaMask(): the shader was not created with alpha mask enabled\n");
+}
+
 }}}
 
 CORRADE_TEST_MAIN(Magnum::Shaders::Test::PhongGLTest)