Shaders: properly use bitangents in Phong normal map calculation.

doc/changelog.dox

@@ -114,6 +114,16 @@ See also:
 -   @ref magnum-sceneconverter "magnum-sceneconverter" now lists also materials
     and textures in `--info`
 
+@subsubsection changelog-latest-changes-shaders Shaders library
+
+-   In the original implementation of normal mapping in @ref Shaders::Phong,
+    there shader didn't provide a way to supply bitangent direction, forcing
+    users to patch normal maps. This is now possible using newly added
+    @ref Shaders::Phong::Tangent4, @ref Shaders::Phong::Bitangent attributes
+    and a @ref Shaders::Phong::Flag::Bitangent flag, implementing support for
+    both four-component tangents (used by glTF, for example) and separate
+    tangent and bitangent direction (used by Assimp).
+
 @subsubsection changelog-latest-changes-trade Trade library
 
 -   Recognizing TIFF file header magic in @ref Trade::AnyImageImporter "AnyImageImporter"

src/Magnum/Shaders/Phong.cpp

@@ -98,6 +98,7 @@ Phong::Phong(const Flags flags, const UnsignedInt lightCount): _flags{flags}, _l
 
     vert.addSource(flags & (Flag::AmbientTexture|Flag::DiffuseTexture|Flag::SpecularTexture|Flag::NormalTexture) ? "#define TEXTURED\n" : "")
         .addSource(flags & Flag::NormalTexture ? "#define NORMAL_TEXTURE\n" : "")
+        .addSource(flags & Flag::Bitangent ? "#define BITANGENT\n" : "")
         .addSource(flags & Flag::VertexColor ? "#define VERTEX_COLOR\n" : "")
         .addSource(flags & Flag::TextureTransformation ? "#define TEXTURE_TRANSFORMATION\n" : "")
         .addSource(Utility::formatString("#define LIGHT_COUNT {}\n", lightCount))
@@ -112,6 +113,7 @@ Phong::Phong(const Flags flags, const UnsignedInt lightCount): _flags{flags}, _l
         .addSource(flags & Flag::DiffuseTexture ? "#define DIFFUSE_TEXTURE\n" : "")
         .addSource(flags & Flag::SpecularTexture ? "#define SPECULAR_TEXTURE\n" : "")
         .addSource(flags & Flag::NormalTexture ? "#define NORMAL_TEXTURE\n" : "")
+        .addSource(flags & Flag::Bitangent ? "#define BITANGENT\n" : "")
         .addSource(flags & Flag::VertexColor ? "#define VERTEX_COLOR\n" : "")
         .addSource(flags & Flag::AlphaMask ? "#define ALPHA_MASK\n" : "")
         #ifndef MAGNUM_TARGET_GLES2
@@ -141,8 +143,11 @@ Phong::Phong(const Flags flags, const UnsignedInt lightCount): _flags{flags}, _l
         bindAttributeLocation(Position::Location, "position");
         if(lightCount)
             bindAttributeLocation(Normal::Location, "normal");
-        if((flags & Flag::NormalTexture) && lightCount)
+        if((flags & Flag::NormalTexture) && lightCount) {
             bindAttributeLocation(Tangent::Location, "tangent");
+            if(flags & Flag::Bitangent)
+                bindAttributeLocation(Bitangent::Location, "bitangent");
+        }
         if(flags & Flag::VertexColor)
             bindAttributeLocation(Color3::Location, "vertexColor"); /* Color4 is the same */
         if(flags & (Flag::AmbientTexture|Flag::DiffuseTexture|Flag::SpecularTexture))
@@ -374,6 +379,7 @@ Debug& operator<<(Debug& debug, const Phong::Flag value) {
         _c(DiffuseTexture)
         _c(SpecularTexture)
         _c(NormalTexture)
+        _c(Bitangent)
         _c(AlphaMask)
         _c(VertexColor)
         _c(TextureTransformation)
@@ -396,6 +402,7 @@ Debug& operator<<(Debug& debug, const Phong::Flags value) {
         Phong::Flag::DiffuseTexture,
         Phong::Flag::SpecularTexture,
         Phong::Flag::NormalTexture,
+        Phong::Flag::Bitangent,
         Phong::Flag::AlphaMask,
         Phong::Flag::VertexColor,
         Phong::Flag::InstancedTextureOffset, /* Superset of TextureTransformation */

src/Magnum/Shaders/Phong.frag

@@ -154,7 +154,12 @@ uniform lowp vec4 lightColors[LIGHT_COUNT]
 #if LIGHT_COUNT
 in mediump vec3 transformedNormal;
 #ifdef NORMAL_TEXTURE
+#ifndef BITANGENT
+in mediump vec4 transformedTangent;
+#else
 in mediump vec3 transformedTangent;
+in mediump vec3 transformedBitangent;
+#endif
 #endif
 in highp vec3 lightDirections[LIGHT_COUNT];
 in highp vec3 cameraDirection;
@@ -218,11 +223,20 @@ void main() {
     /* Normal */
     mediump vec3 normalizedTransformedNormal = normalize(transformedNormal);
     #ifdef NORMAL_TEXTURE
+    #ifndef BITANGENT
+    mediump vec3 normalizedTransformedTangent = normalize(transformedTangent.xyz);
+    #else
     mediump vec3 normalizedTransformedTangent = normalize(transformedTangent);
+    mediump vec3 normalizedTransformedBitangent = normalize(transformedBitangent);
+    #endif
     mediump mat3 tbn = mat3(
         normalizedTransformedTangent,
+        #ifndef BITANGENT
         normalize(cross(normalizedTransformedNormal,
-                        normalizedTransformedTangent)),
+                        normalizedTransformedTangent)*transformedTangent.w),
+        #else
+        normalizedTransformedBitangent,
+        #endif
         normalizedTransformedNormal
     );
     normalizedTransformedNormal = tbn*(normalize((texture(normalTexture, interpolatedTextureCoordinates).rgb*2.0 - vec3(1.0))*vec3(normalTextureScale, normalTextureScale, 1.0)));

src/Magnum/Shaders/Phong.h

@@ -100,6 +100,35 @@ diffuse part and then separate the alpha like this:
 
 @snippet MagnumShaders.cpp Phong-usage-alpha
 
+@section Shaders-Phong-normal-mapping Normal mapping
+
+If you want to use normal textures, enable @ref Flag::NormalTexture and call
+@ref bindNormalTexture(). In addition you need to supply per-vertex tangent and
+bitangent direction:
+
+-   either using a four-component @ref Tangent4 attribute, where the sign of
+    the fourth component defines handedness of tangent basis, as described in
+    @ref Trade::MeshAttribute::Tangent;
+-   or a using pair of three-component @ref Tangent and @ref Bitangent
+    attributes together with enabling @ref Flag::Bitangent
+
+If you supply just a three-component @ref Tangent attribute and no bitangents,
+the shader will implicitly assume the fourth component to be @cpp 1.0f @ce,
+forming a right-handed tangent space. This is a valid optimization when you
+have full control over the bitangent orientation, but won't work with general
+meshes.
+
+@m_class{m-note m-success}
+
+@par
+    You can also use the @ref MeshVisualizer3D shader to visualize and debug
+    per-vertex normal, tangent and binormal direction, among other things.
+
+The strength of the effect can be controlled by
+@ref setNormalTextureScale(). See
+@ref Trade::MaterialAttribute::NormalTextureScale for a description of the
+factor is used.
+
 @section Shaders-Phong-object-id Object ID output
 
 The shader supports writing object ID to the framebuffer for object picking or
@@ -177,11 +206,40 @@ class MAGNUM_SHADERS_EXPORT Phong: public GL::AbstractShaderProgram {
          * @m_since{2019,10}
          *
          * @ref shaders-generic "Generic attribute",
-         * @ref Magnum::Vector3 "Vector3", used only if
+         * @ref Magnum::Vector3 "Vector3". Use either this or the @ref Tangent4
-         * @ref Flag::NormalTexture is set.
+         * attribute. If only a three-component attribute is used and
+         * @ref Flag::Bitangent is not enabled, it's the same as if
+         * @ref Tangent4 was specified with the fourth component always being
+         * @cpp 1.0f @ce. Used only if @ref Flag::NormalTexture is set.
+         * @see @ref Shaders-Phong-normal-mapping
          */
         typedef Generic3D::Tangent Tangent;
 
+        /**
+         * @brief Tangent direction with a bitangent sign
+         * @m_since_latest
+         *
+         * @ref shaders-generic "Generic attribute",
+         * @ref Magnum::Vector4 "Vector4". Use either this or the @ref Tangent
+         * attribute. If @ref Flag::Bitangent is set, the fourth component is
+         * ignored and bitangents are taken from the @ref Bitangent attribute
+         * instead. Used only if @ref Flag::NormalTexture is set.
+         * @see @ref Shaders-Phong-normal-mapping
+         */
+        typedef typename Generic3D::Tangent4 Tangent4;
+
+        /**
+         * @brief Bitangent direction
+         * @m_since_latest
+         *
+         * @ref shaders-generic "Generic attribute",
+         * @ref Magnum::Vector3 "Vector3". Use either this or the @ref Tangent4
+         * attribute. Used only if both @ref Flag::NormalTexture and
+         * @ref Flag::Bitangent are set.
+         * @see @ref Shaders-Phong-normal-mapping
+         */
+        typedef typename Generic3D::Bitangent Bitangent;
+
         /**
          * @brief 2D texture coordinates
          *
@@ -348,6 +406,15 @@ class MAGNUM_SHADERS_EXPORT Phong: public GL::AbstractShaderProgram {
              */
             VertexColor = 1 << 5,
 
+            /**
+             * Use the separate @ref Bitangent attribute for retrieving vertex
+             * bitangents. If this flag is not present, the last component of
+             * @ref Tangent4 is used to calculate bitangent direction. See
+             * @ref Shaders-Phong-normal-mapping for more information.
+             * @m_since_latest
+             */
+            Bitangent = 1 << 11,
+
             /**
              * Enable texture coordinate transformation. If this flag is set,
              * the shader expects that at least one of

src/Magnum/Shaders/Phong.vert

@@ -95,7 +95,20 @@ in mediump vec3 normal;
 #ifdef EXPLICIT_ATTRIB_LOCATION
 layout(location = TANGENT_ATTRIBUTE_LOCATION)
 #endif
-in mediump vec3 tangent;
+in mediump
+    #ifndef BITANGENT
+    vec4
+    #else
+    vec3
+    #endif
+    tangent;
+#endif
+
+#ifdef BITANGENT
+#ifdef EXPLICIT_ATTRIB_LOCATION
+layout(location = BITANGENT_ATTRIBUTE_LOCATION)
+#endif
+in mediump vec3 bitangent;
 #endif
 #endif
 
@@ -148,7 +161,12 @@ in mediump vec2 instancedTextureOffset;
 #if LIGHT_COUNT
 out mediump vec3 transformedNormal;
 #ifdef NORMAL_TEXTURE
+#ifndef BITANGENT
+out mediump vec4 transformedTangent;
+#else
 out mediump vec3 transformedTangent;
+out mediump vec3 transformedBitangent;
+#endif
 #endif
 out highp vec3 lightDirections[LIGHT_COUNT];
 out highp vec3 cameraDirection;
@@ -171,11 +189,24 @@ void main() {
         #endif
         normal;
     #ifdef NORMAL_TEXTURE
+    #ifndef BITANGENT
+    transformedTangent = vec4(normalMatrix*
+        #ifdef INSTANCED_TRANSFORMATION
+        instancedNormalMatrix*
+        #endif
+        tangent.xyz, tangent.w);
+    #else
     transformedTangent = normalMatrix*
         #ifdef INSTANCED_TRANSFORMATION
         instancedNormalMatrix*
         #endif
         tangent;
+    transformedBitangent = normalMatrix*
+        #ifdef INSTANCED_TRANSFORMATION
+        instancedNormalMatrix*
+        #endif
+        bitangent;
+    #endif
     #endif
 
     /* Direction to the light */

src/Magnum/Shaders/Test/CMakeLists.txt

@@ -234,6 +234,7 @@ if(BUILD_GL_TESTS)
             PhongTestFiles/textured-diffuse.tga
             PhongTestFiles/textured-diffuse-transformed.tga
             PhongTestFiles/textured-normal.tga
+            PhongTestFiles/textured-normal-left.tga
             PhongTestFiles/textured-normal0.0.tga
             PhongTestFiles/textured-normal0.5.tga
             PhongTestFiles/textured-specular.tga

src/Magnum/Shaders/Test/PhongGLTest.cpp

@@ -145,6 +145,8 @@ constexpr struct {
     {"diffuse texture + texture transform", Phong::Flag::DiffuseTexture|Phong::Flag::TextureTransformation, 1},
     {"specular texture", Phong::Flag::SpecularTexture, 1},
     {"normal texture", Phong::Flag::NormalTexture, 1},
+    {"normal texture + separate bitangents", Phong::Flag::NormalTexture|Phong::Flag::Bitangent, 1},
+    {"separate bitangents alone", Phong::Flag::Bitangent, 1},
     {"ambient + diffuse texture", Phong::Flag::AmbientTexture|Phong::Flag::DiffuseTexture, 1},
     {"ambient + specular texture", Phong::Flag::AmbientTexture|Phong::Flag::SpecularTexture, 1},
     {"diffuse + specular texture", Phong::Flag::DiffuseTexture|Phong::Flag::SpecularTexture, 1},
@@ -210,13 +212,52 @@ const struct {
     bool multiBind;
     Deg rotation;
     Float scale;
+    Vector4 tangent;
+    Vector3 bitangent;
+    Shaders::Phong::Tangent4::Components tangentComponents;
+    bool flipNormalY;
+    Shaders::Phong::Flags flags;
 } RenderTexturedNormalData[]{
-    {"", "textured-normal.tga", false, {}, 1.0f},
+    {"", "textured-normal.tga", false, {}, 1.0f,
-    {"multi bind", "textured-normal.tga", true, {}, 1.0f},
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
-    {"rotated 90°", "textured-normal.tga", false, 90.0_degf, 1.0f},
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
-    {"rotated -90°", "textured-normal.tga", false, -90.0_degf, 1.0f},
+    {"multi bind", "textured-normal.tga", true, {}, 1.0f,
-    {"0.5 scale", "textured-normal0.5.tga", false, {}, 0.5f},
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
-    {"0.0 scale", "textured-normal0.0.tga", false, {}, 0.0f}
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
+    {"rotated 90°", "textured-normal.tga", false, 90.0_degf, 1.0f,
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
+    {"rotated -90°", "textured-normal.tga", false, -90.0_degf, 1.0f,
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
+    {"0.5 scale", "textured-normal0.5.tga", false, {}, 0.5f,
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
+    {"0.0 scale", "textured-normal0.0.tga", false, {}, 0.0f,
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
+    /* The fourth component, if missing, gets automatically filled up to 1,
+       so this should work */
+    {"implicit bitangent direction", "textured-normal.tga", false, {}, 1.0f,
+        {1.0f, 0.0f, 0.0f, 0.0f}, {},
+        Shaders::Phong::Tangent4::Components::Three, false, {}},
+    {"separate bitangents", "textured-normal.tga", false, {}, 1.0f,
+        {1.0f, 0.0f, 0.0f, 1.0f}, {0.0f, 1.0f, 0.0f},
+        Shaders::Phong::Tangent4::Components::Three, false,
+        Shaders::Phong::Flag::Bitangent},
+    {"right-handed, flipped Y", "textured-normal-left.tga", false, {}, 1.0f,
+        {1.0f, 0.0f, 0.0f, 1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, true, {}},
+    {"left-handed", "textured-normal-left.tga", false, {}, 1.0f,
+        {1.0f, 0.0f, 0.0f, -1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, false, {}},
+    {"left-handed, separate bitangents", "textured-normal-left.tga", false, {}, 1.0f,
+        {1.0f, 0.0f, 0.0f, 0.0f}, {0.0f, -1.0f, 0.0f},
+        Shaders::Phong::Tangent4::Components::Three, false,
+        Shaders::Phong::Flag::Bitangent},
+    {"left-handed, flipped Y", "textured-normal.tga", false, {}, 1.0f,
+        {1.0f, 0.0f, 0.0f, -1.0f}, {},
+        Shaders::Phong::Tangent4::Components::Four, true, {}}
 };
 
 const struct {
@@ -903,9 +944,14 @@ void PhongGLTest::renderTexturedNormal() {
     Containers::Pointer<Trade::AbstractImporter> importer = _manager.loadAndInstantiate("AnyImageImporter");
     CORRADE_VERIFY(importer);
 
-    GL::Texture2D normal;
+    /* Normal texture. Flip normal Y, if requested */
     Containers::Optional<Trade::ImageData2D> image;
     CORRADE_VERIFY(importer->openFile(Utility::Directory::join(_testDir, "TestFiles/normal-texture.tga")) && (image = importer->image2D(0)));
+    if(data.flipNormalY) for(auto row: image->mutablePixels<Color3ub>())
+        for(Color3ub& pixel: row)
+            pixel.y() = 255 - pixel.y();
+
+    GL::Texture2D normal;
     normal.setMinificationFilter(GL::SamplerFilter::Linear)
         .setMagnificationFilter(GL::SamplerFilter::Linear)
         .setWrapping(GL::SamplerWrapping::ClampToEdge)
@@ -914,16 +960,24 @@ void PhongGLTest::renderTexturedNormal() {
 
     GL::Mesh plane = MeshTools::compile(Primitives::planeSolid( Primitives::PlaneFlag::TextureCoordinates));
 
-    /* Add hardcoded tangents */
+    /* Add tangents / bitangents of desired component count. Unused components
-    /** @todo remove once MeshData is sane */
+       are set to zero to ensure the shader doesn't use them. */
+    const struct TangentBitangent {
+        Vector4 tangent;
+        Vector3 bitangent;
+    } tangentBitangent{data.tangent, data.bitangent};
     GL::Buffer tangents;
-    tangents.setData(Containers::Array<Vector3>{Containers::DirectInit, 4, Vector3::xAxis()});
+    tangents.setData(Containers::Array<TangentBitangent>{Containers::DirectInit, 4, tangentBitangent});
-    plane.addVertexBuffer(std::move(tangents), 0, Shaders::Phong::Tangent{});
+    plane.addVertexBuffer(tangents, 0, sizeof(TangentBitangent),
+        GL::DynamicAttribute{Shaders::Phong::Tangent4{data.tangentComponents}});
+    plane.addVertexBuffer(std::move(tangents), sizeof(Vector4),
+        sizeof(TangentBitangent),
+        GL::DynamicAttribute{Shaders::Phong::Bitangent{}});
 
     /* Rotating the view a few times (together with light positions). If the
        tangent transformation in the shader is correct, it should result in
        exactly the same images. */
-    Phong shader{Phong::Flag::NormalTexture, 2};
+    Phong shader{Phong::Flag::NormalTexture|data.flags, 2};
     shader.setLightPositions({
             Matrix4::rotationZ(data.rotation).transformPoint({-3.0f, -3.0f, 0.0f}),
             Matrix4::rotationZ(data.rotation).transformPoint({ 3.0f, -3.0f, 0.0f})})