Basic async shader compilation for FlatGL

src/Magnum/GL/AbstractShaderProgram.cpp

@@ -586,10 +586,21 @@ void AbstractShaderProgram::transformFeedbackVaryingsImplementationDanglingWorka
 bool AbstractShaderProgram::link() { return link({*this}); }
 
 bool AbstractShaderProgram::link(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders) {
-    bool allSuccess = true;
+    submitLink(shaders);
+    return checkLink(shaders);
+}
+
+void AbstractShaderProgram::submitLink() { submitLink({*this}); }
 
-    /* Invoke (possibly parallel) linking on all shaders */
+bool AbstractShaderProgram::checkLink() { return checkLink({*this}); }
+
+void AbstractShaderProgram::submitLink(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders) {
+     /* Invoke (possibly parallel) linking on all shaders */
     for(AbstractShaderProgram& shader: shaders) glLinkProgram(shader._id);
+}
+
+bool AbstractShaderProgram::checkLink(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders) {
+    bool allSuccess = true;
 
     /* After linking phase, check status of all shaders */
     Int i = 1;
@@ -632,6 +643,16 @@ bool AbstractShaderProgram::link(std::initializer_list<Containers::Reference<Abs
     return allSuccess;
 }
 
+bool AbstractShaderProgram::isLinkFinished() {
+    GLint success;
+    glGetProgramiv(_id, GL_COMPLETION_STATUS_KHR, &success);
+    return success == GL_TRUE;
+}
+
+bool AbstractShaderProgram::CompileState::isLinkFinished() {
+    return AbstractShaderProgram::isLinkFinished();
+}
+
 void AbstractShaderProgram::cleanLogImplementationNoOp(std::string&) {}
 
 #if defined(CORRADE_TARGET_WINDOWS) && !defined(MAGNUM_TARGET_GLES)

src/Magnum/GL/AbstractShaderProgram.h

@@ -449,6 +449,8 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
     friend Implementation::ShaderProgramState;
 
     public:
+        struct CompileState;
+
         #ifndef MAGNUM_TARGET_GLES2
         /**
          * @brief Buffer mode for transform feedback
@@ -1260,7 +1262,7 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
 
     protected:
         /**
-         * @brief Link the shader
+         * @brief Link the shaders
          *
          * Returns @cpp false @ce if linking of any shader failed, @cpp true @ce
          * if everything succeeded. Linker message (if any) is printed to error
@@ -1274,6 +1276,26 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
          */
         static bool link(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders);
 
+        /**
+         * @brief Submit shaders for linking
+         *
+         * The operation is batched in a
+         * way that allows the driver to link multiple shaders simultaneously
+         * (i.e. in multiple threads).
+         *
+         */
+        static void submitLink(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders);
+
+        /**
+         * @brief Check linking status of shaders and await completion
+         *
+         * Returns @cpp false @ce if linking of any shader failed, @cpp true @ce
+         * if everything succeeded. Linker message (if any) is printed to error
+         * output.
+         *
+         */
+        static bool checkLink(std::initializer_list<Containers::Reference<AbstractShaderProgram>> shaders);
+
         #if !defined(MAGNUM_TARGET_GLES2) && !defined(MAGNUM_TARGET_WEBGL)
         /**
          * @brief Allow retrieving program binary
@@ -1453,6 +1475,24 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
          */
         bool link();
 
+        /**
+         * @brief Submit single shader for linking
+         *
+         */
+        void submitLink();
+
+        /**
+         * @brief Check link status and await completion
+         *
+         */
+        bool checkLink();
+
+        /**
+         * @brief Non-blocking linking status check
+         *
+         */
+        bool isLinkFinished();
+
         /**
          * @brief Get uniform location
          * @param name          Uniform name
@@ -1729,6 +1769,11 @@ class MAGNUM_GL_EXPORT AbstractShaderProgram: public AbstractObject {
         #endif
 };
 
+struct AbstractShaderProgram::CompileState : public AbstractShaderProgram {
+    using AbstractShaderProgram::AbstractShaderProgram;
+    bool isLinkFinished();
+};
+
 }}
 
 #endif

src/Magnum/GL/Shader.cpp

@@ -749,14 +749,27 @@ Shader& Shader::addFile(const std::string& filename) {
 
 bool Shader::compile() { return compile({*this}); }
 
+void Shader::submitCompile() { submitCompile({*this}); }
+
+bool Shader::checkCompile() { return checkCompile({*this}); }
+
 bool Shader::compile(std::initializer_list<Containers::Reference<Shader>> shaders) {
-    bool allSuccess = true;
+    submitCompile(shaders);
+    return checkCompile(shaders);
+}
 
+bool Shader::isCompileFinished() {
+    GLint success;
+    glGetShaderiv(_id, GL_COMPLETION_STATUS_KHR, &success);
+    return success == GL_TRUE;
+}
+
+void Shader::submitCompile(std::initializer_list<Containers::Reference<Shader>> shaders) {
     /* Allocate large enough array for source pointers and sizes (to avoid
        reallocating it for each of them) */
     std::size_t maxSourceCount = 0;
     for(Shader& shader: shaders) {
-        CORRADE_ASSERT(shader._sources.size() > 1, "GL::Shader::compile(): no files added", false);
+        CORRADE_ASSERT(shader._sources.size() > 1, "GL::Shader::compile(): no files added", );
         maxSourceCount = Math::max(shader._sources.size(), maxSourceCount);
     }
     /** @todo ArrayTuple/VLAs */
@@ -775,6 +788,10 @@ bool Shader::compile(std::initializer_list<Containers::Reference<Shader>> shader
 
     /* Invoke (possibly parallel) compilation on all shaders */
     for(Shader& shader: shaders) glCompileShader(shader._id);
+}
+
+bool Shader::checkCompile(std::initializer_list<Containers::Reference<Shader>> shaders) {
+    bool allSuccess = true;
 
     /* After compilation phase, check status of all shaders */
     Int i = 1;

src/Magnum/GL/Shader.h

@@ -521,6 +521,23 @@ class MAGNUM_GL_EXPORT Shader: public AbstractObject {
          */
         static bool compile(std::initializer_list<Containers::Reference<Shader>> shaders);
 
+        /**
+         * @brief Submit multiple shaders for compilation
+         *
+         * The operation is batched in a way that
+         * allows the driver to perform multiple compilations simultaneously
+         * (i.e. in multiple threads).
+         */
+        static void submitCompile(std::initializer_list<Containers::Reference<Shader>> shaders);
+
+        /**
+         * @brief Check compilations status of multiple shaders and await completion
+         *
+         * Returns @cpp false @ce if compilation of any shader failed,
+         * @cpp true @ce if everything succeeded.
+         */
+        static bool checkCompile(std::initializer_list<Containers::Reference<Shader>> shaders);
+
         /**
          * @brief Constructor
          * @param version   Target version
@@ -643,6 +660,25 @@ class MAGNUM_GL_EXPORT Shader: public AbstractObject {
          */
         bool compile();
 
+        /**
+         * @brief Submit shader for compilation
+         *
+         * Submits shader for compilation.
+         */
+        void submitCompile();
+
+        /**
+         * @brief Check compilation status and await completion
+         *
+         */
+        bool checkCompile();
+
+        /**
+         * @brief Non-blocking compilation status check
+         *
+         */
+        bool isCompileFinished();
+
     private:
         void MAGNUM_GL_LOCAL addSourceImplementationDefault(std::string source);
         #if defined(CORRADE_TARGET_EMSCRIPTEN) && defined(__EMSCRIPTEN_PTHREADS__)

src/Magnum/Shaders/FlatGL.cpp

@@ -31,7 +31,6 @@
 
 #include "Magnum/GL/Context.h"
 #include "Magnum/GL/Extensions.h"
-#include "Magnum/GL/Shader.h"
 #include "Magnum/GL/Texture.h"
 #include "Magnum/Math/Color.h"
 #include "Magnum/Math/Matrix3.h"
@@ -68,16 +67,11 @@ namespace {
     #endif
 }
 
-template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags
-    #ifndef MAGNUM_TARGET_GLES2
-    , const UnsignedInt materialCount, const UnsignedInt drawCount
-    #endif
-):
-    _flags{flags}
-    #ifndef MAGNUM_TARGET_GLES2
-    , _materialCount{materialCount}, _drawCount{drawCount}
-    #endif
-{
+template<UnsignedInt dimensions> typename FlatGL<dimensions>::CompileState FlatGL<dimensions>::compile(Flags flags
+#ifndef MAGNUM_TARGET_GLES2
+, UnsignedInt materialCount, UnsignedInt drawCount
+#endif
+) {
     #ifndef CORRADE_NO_ASSERT
     {
         const bool textureTransformationNotEnabledOrTextured = !(flags & Flag::TextureTransformation) || flags & Flag::Textured
@@ -86,22 +80,22 @@ template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags
             #endif
             ;
         CORRADE_ASSERT(textureTransformationNotEnabledOrTextured,
-            "Shaders::FlatGL: texture transformation enabled but the shader is not textured", );
+            "Shaders::FlatGL: texture transformation enabled but the shader is not textured", CompileState{NoCreate});
     }
     #endif
 
     #ifndef MAGNUM_TARGET_GLES2
     CORRADE_ASSERT(!(flags >= Flag::UniformBuffers) || materialCount,
-        "Shaders::FlatGL: material count can't be zero", );
+        "Shaders::FlatGL: material count can't be zero", CompileState{NoCreate});
     CORRADE_ASSERT(!(flags >= Flag::UniformBuffers) || drawCount,
-        "Shaders::FlatGL: draw count can't be zero", );
+        "Shaders::FlatGL: draw count can't be zero", CompileState{NoCreate});
     #endif
 
     #ifndef MAGNUM_TARGET_GLES2
     CORRADE_ASSERT(!(flags & Flag::TextureArrays) || flags & Flag::Textured || flags >= Flag::ObjectIdTexture,
-        "Shaders::FlatGL: texture arrays enabled but the shader is not textured", );
+        "Shaders::FlatGL: texture arrays enabled but the shader is not textured", CompileState{NoCreate});
     CORRADE_ASSERT(!(flags & Flag::UniformBuffers) || !(flags & Flag::TextureArrays) || flags >= (Flag::TextureArrays|Flag::TextureTransformation),
-        "Shaders::FlatGL: texture arrays require texture transformation enabled as well if uniform buffers are used", );
+        "Shaders::FlatGL: texture arrays require texture transformation enabled as well if uniform buffers are used", CompileState{NoCreate});
     #endif
 
     #ifndef MAGNUM_TARGET_GLES
@@ -195,9 +189,14 @@ template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags
     frag.addSource(rs.getString("generic.glsl"))
         .addSource(rs.getString("Flat.frag"));
 
-    CORRADE_INTERNAL_ASSERT_OUTPUT(GL::Shader::compile({vert, frag}));
+    GL::Shader::submitCompile({vert, frag});
 
-    attachShaders({vert, frag});
+    CompileState cs{std::move(frag), std::move(vert), flags
+    #ifndef MAGNUM_TARGET_GLES2
+    , materialCount, drawCount
+    #endif
+    };
+    cs.attachShaders({cs._frag, cs._vert});
 
     /* ES3 has this done in the shader directly and doesn't even provide
        bindFragmentDataLocation() */
@@ -206,53 +205,71 @@ template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags
     if(!context.isExtensionSupported<GL::Extensions::ARB::explicit_attrib_location>(version))
     #endif
     {
-        bindAttributeLocation(Position::Location, "position");
+        cs.bindAttributeLocation(Position::Location, "position");
         if(flags & Flag::Textured
             #ifndef MAGNUM_TARGET_GLES2
             || flags >= Flag::ObjectIdTexture
             #endif
         )
-            bindAttributeLocation(TextureCoordinates::Location, "textureCoordinates");
+            cs.bindAttributeLocation(TextureCoordinates::Location, "textureCoordinates");
         if(flags & Flag::VertexColor)
-            bindAttributeLocation(Color3::Location, "vertexColor"); /* Color4 is the same */
+            cs.bindAttributeLocation(Color3::Location, "vertexColor"); /* Color4 is the same */
         #ifndef MAGNUM_TARGET_GLES2
         if(flags & Flag::ObjectId) {
-            bindFragmentDataLocation(ColorOutput, "color");
-            bindFragmentDataLocation(ObjectIdOutput, "objectId");
+            cs.bindFragmentDataLocation(ColorOutput, "color");
+            cs.bindFragmentDataLocation(ObjectIdOutput, "objectId");
         }
         if(flags >= Flag::InstancedObjectId)
-            bindAttributeLocation(ObjectId::Location, "instanceObjectId");
+            cs.bindAttributeLocation(ObjectId::Location, "instanceObjectId");
         #endif
         if(flags & Flag::InstancedTransformation)
-            bindAttributeLocation(TransformationMatrix::Location, "instancedTransformationMatrix");
+            cs.bindAttributeLocation(TransformationMatrix::Location, "instancedTransformationMatrix");
         if(flags >= Flag::InstancedTextureOffset)
-            bindAttributeLocation(TextureOffset::Location, "instancedTextureOffset");
+            cs.bindAttributeLocation(TextureOffset::Location, "instancedTextureOffset");
     }
     #endif
 
-    CORRADE_INTERNAL_ASSERT_OUTPUT(link());
+    cs.submitLink();
+
+    return cs;
+}
+
+template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(CompileState&& cs)
+: AbstractShaderProgram{static_cast<AbstractShaderProgram&&>(std::move(cs))},
+    _flags(cs._flags), _materialCount{cs._materialCount}, _drawCount(cs._drawCount) {
+
+    CORRADE_INTERNAL_ASSERT_OUTPUT(GL::Shader::checkCompile({cs._vert, cs._frag}));
+    CORRADE_INTERNAL_ASSERT_OUTPUT(checkLink());
+
+    const GL::Context& context = GL::Context::current();
+
+    #ifndef MAGNUM_TARGET_GLES
+    const GL::Version version = context.supportedVersion({GL::Version::GL320, GL::Version::GL310, GL::Version::GL300, GL::Version::GL210});
+    #else
+    const GL::Version version = context.supportedVersion({GL::Version::GLES300, GL::Version::GLES200});
+    #endif
 
     #ifndef MAGNUM_TARGET_GLES
     if(!context.isExtensionSupported<GL::Extensions::ARB::explicit_uniform_location>(version))
     #endif
     {
         #ifndef MAGNUM_TARGET_GLES2
-        if(flags >= Flag::UniformBuffers) {
+        if(_flags >= Flag::UniformBuffers) {
             if(_drawCount > 1) _drawOffsetUniform = uniformLocation("drawOffset");
         } else
         #endif
         {
             _transformationProjectionMatrixUniform = uniformLocation("transformationProjectionMatrix");
-            if(flags & Flag::TextureTransformation)
+            if(_flags & Flag::TextureTransformation)
                 _textureMatrixUniform = uniformLocation("textureMatrix");
             #ifndef MAGNUM_TARGET_GLES2
-            if(flags & Flag::TextureArrays)
+            if(_flags & Flag::TextureArrays)
                 _textureLayerUniform = uniformLocation("textureLayer");
             #endif
             _colorUniform = uniformLocation("color");
-            if(flags & Flag::AlphaMask) _alphaMaskUniform = uniformLocation("alphaMask");
+            if(_flags & Flag::AlphaMask) _alphaMaskUniform = uniformLocation("alphaMask");
             #ifndef MAGNUM_TARGET_GLES2
-            if(flags & Flag::ObjectId) _objectIdUniform = uniformLocation("objectId");
+            if(_flags & Flag::ObjectId) _objectIdUniform = uniformLocation("objectId");
             #endif
         }
     }
@@ -261,13 +278,13 @@ template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags
     if(!context.isExtensionSupported<GL::Extensions::ARB::shading_language_420pack>(version))
     #endif
     {
-        if(flags & Flag::Textured) setUniform(uniformLocation("textureData"), TextureUnit);
+        if(_flags & Flag::Textured) setUniform(uniformLocation("textureData"), TextureUnit);
         #ifndef MAGNUM_TARGET_GLES2
-        if(flags >= Flag::ObjectIdTexture) setUniform(uniformLocation("objectIdTextureData"), ObjectIdTextureUnit);
-        if(flags >= Flag::UniformBuffers) {
+        if(_flags >= Flag::ObjectIdTexture) setUniform(uniformLocation("objectIdTextureData"), ObjectIdTextureUnit);
+        if(_flags >= Flag::UniformBuffers) {
             setUniformBlockBinding(uniformBlockIndex("TransformationProjection"), TransformationProjectionBufferBinding);
             setUniformBlockBinding(uniformBlockIndex("Draw"), DrawBufferBinding);
-            if(flags & Flag::TextureTransformation)
+            if(_flags & Flag::TextureTransformation)
                 setUniformBlockBinding(uniformBlockIndex("TextureTransformation"), TextureTransformationBufferBinding);
             setUniformBlockBinding(uniformBlockIndex("Material"), MaterialBufferBinding);
         }
@@ -277,22 +294,34 @@ template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags
     /* Set defaults in OpenGL ES (for desktop they are set in shader code itself) */
     #ifdef MAGNUM_TARGET_GLES
     #ifndef MAGNUM_TARGET_GLES2
-    if(flags >= Flag::UniformBuffers) {
+    if(_flags >= Flag::UniformBuffers) {
         /* Draw offset is zero by default */
     } else
     #endif
     {
         setTransformationProjectionMatrix(MatrixTypeFor<dimensions, Float>{Math::IdentityInit});
-        if(flags & Flag::TextureTransformation)
+        if(_flags & Flag::TextureTransformation)
             setTextureMatrix(Matrix3{Math::IdentityInit});
         /* Texture layer is zero by default */
         setColor(Magnum::Color4{1.0f});
-        if(flags & Flag::AlphaMask) setAlphaMask(0.5f);
+        if(_flags & Flag::AlphaMask) setAlphaMask(0.5f);
         /* Object ID is zero by default */
     }
     #endif
 }
 
+template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(Flags flags
+    #ifndef MAGNUM_TARGET_GLES2
+    , UnsignedInt materialCount, UnsignedInt drawCount
+    #endif
+):
+    #ifndef MAGNUM_TARGET_GLES2
+    FlatGL(compile(flags, materialCount, drawCount))
+    #else
+    FlatGL{compile(flags)}
+    #endif
+{}
+
 #ifndef MAGNUM_TARGET_GLES2
 template<UnsignedInt dimensions> FlatGL<dimensions>::FlatGL(const Flags flags): FlatGL{flags, 1, 1} {}
 #endif

src/Magnum/Shaders/FlatGL.h

@@ -32,6 +32,7 @@
 
 #include "Magnum/DimensionTraits.h"
 #include "Magnum/GL/AbstractShaderProgram.h"
+#include "Magnum/GL/Shader.h"
 #include "Magnum/Shaders/GenericGL.h"
 #include "Magnum/Shaders/visibility.h"
 
@@ -602,6 +603,48 @@ template<UnsignedInt dimensions> class MAGNUM_SHADERS_EXPORT FlatGL: public GL::
          */
         explicit FlatGL(NoCreateT) noexcept: GL::AbstractShaderProgram{NoCreate} {}
 
+        struct CompileState;
+
+        explicit FlatGL(CompileState&& cs);
+
+        static CompileState compile(Flags flags
+        #ifndef MAGNUM_TARGET_GLES2
+        , UnsignedInt materialCount, UnsignedInt drawCount
+        #endif
+        );
+
+        struct CompileState : public GL::AbstractShaderProgram::CompileState {
+            friend CompileState FlatGL::compile(Flags
+            #ifndef MAGNUM_TARGET_GLES2
+            , UnsignedInt, UnsignedInt
+            #endif
+            );
+            friend FlatGL::FlatGL(CompileState&&);
+
+            explicit CompileState(NoCreateT) noexcept: GL::AbstractShaderProgram::CompileState({NoCreate}), 
+                _vert(NoCreate), _frag(NoCreate) {}
+
+            CompileState(GL::Shader&& vert, GL::Shader&& frag, Flags flags
+                #ifndef MAGNUM_TARGET_GLES2
+                , UnsignedInt materialCount, UnsignedInt drawCount
+                #endif
+                )
+                : _vert(std::move(vert)), _frag(std::move(frag)), _flags(flags) 
+                #ifndef MAGNUM_TARGET_GLES2
+                ,_materialCount(materialCount), _drawCount(drawCount) 
+                #endif
+                {}
+
+        private:
+            GL::Shader _vert, _frag;
+            Flags _flags;
+            
+            #ifndef MAGNUM_TARGET_GLES2
+            UnsignedInt _materialCount;
+            UnsignedInt _drawCount;
+            #endif
+        };
+
         /** @brief Copying is not allowed */
         FlatGL(const FlatGL<dimensions>&) = delete;
 

src/MagnumExternal/OpenGL/GL/flextGL.h

@@ -1523,7 +1523,7 @@ typedef void (APIENTRY *GLDEBUGPROC)(GLenum source,GLenum type,GLuint id,GLenum
 
 /* GL_VERSION_4_5 */
 
-#define GL_CONTEXT_LOST 0x0507
+#define GL_CONTEXT_LOST 0x0507 
 #define GL_LOWER_LEFT 0x8CA1
 #define GL_UPPER_LEFT 0x8CA2
 #define GL_NEGATIVE_ONE_TO_ONE 0x935E
@@ -1773,6 +1773,10 @@ typedef void (APIENTRY *GLDEBUGPROC)(GLenum source,GLenum type,GLuint id,GLenum
 
 #define GL_BLEND_ADVANCED_COHERENT_KHR 0x9285
 
+/* GL_KHR_parallel_shader_compile */
+
+#define GL_COMPLETION_STATUS_KHR 0x91B1 
+
 /* GL_NV_sample_locations */
 
 #define GL_SAMPLE_LOCATION_SUBPIXEL_BITS_NV 0x933D