From 7b504a9bb1165393d192d096000c580035adc8ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Vondru=C5=A1?= Date: Mon, 24 Mar 2014 23:44:07 +0100 Subject: [PATCH] Finally implemented parallelizable shader compilation and linking. As g_truc said long ago: https://twitter.com/g_truc/status/352778836657700866 Currently there is not much use of this as the stock shaders are compiled one by one (and doing it differently would make things needlessly overcomplicated), but the users can do parallel compilation of their own shaders. Also removed a bunch of now-unneeded TODOs and made the linker/compiler code nearly similar. Also the whole Shader::compile() call now does two allocations in total instead of two allocations for each shader. --- src/Magnum/AbstractShaderProgram.cpp | 77 ++++++++++-------- src/Magnum/AbstractShaderProgram.h | 30 ++++--- src/Magnum/Shader.cpp | 112 ++++++++++++++++----------- src/Magnum/Shader.h | 28 +++++-- 4 files changed, 152 insertions(+), 95 deletions(-) diff --git a/src/Magnum/AbstractShaderProgram.cpp b/src/Magnum/AbstractShaderProgram.cpp index 4d1e966cf..1709edfb7 100644 --- a/src/Magnum/AbstractShaderProgram.cpp +++ b/src/Magnum/AbstractShaderProgram.cpp @@ -269,40 +269,53 @@ void AbstractShaderProgram::bindFragmentDataLocationIndexed(UnsignedInt location } #endif -bool AbstractShaderProgram::link() { - /* Link shader program */ - glLinkProgram(_id); - - /* Check link status */ - GLint success, logLength; - glGetProgramiv(_id, GL_LINK_STATUS, &success); - glGetProgramiv(_id, GL_INFO_LOG_LENGTH, &logLength); - - /* Error or warning message. The string is returned null-terminated, scrap - the \0 at the end afterwards */ - std::string message(logLength, '\n'); - if(message.size() > 1) - glGetProgramInfoLog(_id, message.size(), nullptr, &message[0]); - message.resize(std::max(logLength, 1)-1); - - /* Show error log and delete shader */ - if(!success) { - Error out; - out.setFlag(Debug::NewLineAtTheEnd, false); - out.setFlag(Debug::SpaceAfterEachValue, false); - out << "AbstractShaderProgram: linking failed with the following message:\n" - << message; - - /* Or just warnings, if there are any */ - } else if(!message.empty()) { - Debug out; - out.setFlag(Debug::NewLineAtTheEnd, false); - out.setFlag(Debug::SpaceAfterEachValue, false); - out << "AbstractShaderProgram: linking succeeded with the following message:\n" - << message; +bool AbstractShaderProgram::link(std::initializer_list> shaders) { + bool allSuccess = true; + + /* Invoke (possibly parallel) linking on all shaders */ + for(AbstractShaderProgram& shader: shaders) glLinkProgram(shader._id); + + /* After linking phase, check status of all shaders */ + Int i = 1; + for(AbstractShaderProgram& shader: shaders) { + GLint success, logLength; + glGetProgramiv(shader._id, GL_LINK_STATUS, &success); + glGetProgramiv(shader._id, GL_INFO_LOG_LENGTH, &logLength); + + /* Error or warning message. The string is returned null-terminated, + scrap the \0 at the end afterwards */ + std::string message(logLength, '\n'); + if(message.size() > 1) + glGetProgramInfoLog(shader._id, message.size(), nullptr, &message[0]); + message.resize(std::max(logLength, 1)-1); + + /* Show error log */ + if(!success) { + Error out; + out.setFlag(Debug::NewLineAtTheEnd, false); + out.setFlag(Debug::SpaceAfterEachValue, false); + out << "AbstractShaderProgram::link(): linking"; + if(shaders.size() != 1) out << " of shader " << std::to_string(i); + out << " failed with the following message:\n" + << message; + + /* Or just warnings, if any */ + } else if(!message.empty()) { + Debug out; + out.setFlag(Debug::NewLineAtTheEnd, false); + out.setFlag(Debug::SpaceAfterEachValue, false); + out << "AbstractShaderProgram::link(): linking"; + if(shaders.size() != 1) out << " of shader " << std::to_string(i); + out << " succeeded with the following message:\n" + << message; + } + + /* Success of all depends on each of them */ + allSuccess = allSuccess && success; + ++i; } - return success; + return allSuccess; } Int AbstractShaderProgram::uniformLocation(const std::string& name) { diff --git a/src/Magnum/AbstractShaderProgram.h b/src/Magnum/AbstractShaderProgram.h index 138ce598a..124625d4c 100644 --- a/src/Magnum/AbstractShaderProgram.h +++ b/src/Magnum/AbstractShaderProgram.h @@ -29,6 +29,7 @@ * @brief Class @ref Magnum::AbstractShaderProgram */ +#include #include #include @@ -320,8 +321,6 @@ comes in handy. @see @ref portability-shaders @todo Use Containers::ArrayReference for setting uniform arrays? -@todo Compiling and linking more than one shader in parallel, then checking - status, should be faster -- https://twitter.com/g_truc/status/352778836657700866 @todo `GL_NUM_{PROGRAM,SHADER}_BINARY_FORMATS` + `GL_{PROGRAM,SHADER}_BINARY_FORMATS` (vector), (@extension{ARB,ES2_compatibility}) */ class MAGNUM_EXPORT AbstractShaderProgram: public AbstractObject { @@ -562,6 +561,21 @@ class MAGNUM_EXPORT AbstractShaderProgram: public AbstractObject { #endif protected: + /** + * @brief Link the shader + * + * Returns `false` if linking of any shader failed, `true` if + * everything succeeded. Linker message (if any) is printed to error + * output. All attached shaders must be compiled with + * @ref Shader::compile() before linking. The operation is batched in a + * way that allows the driver to link multiple shaders simultaenously + * (i.e. in multiple threads). + * @see @fn_gl{LinkProgram}, @fn_gl{GetProgram} with + * @def_gl{LINK_STATUS} and @def_gl{INFO_LOG_LENGTH}, + * @fn_gl{GetProgramInfoLog} + */ + static bool link(std::initializer_list> shaders); + #ifndef MAGNUM_TARGET_GLES2 /** * @brief Allow retrieving program binary @@ -655,14 +669,12 @@ class MAGNUM_EXPORT AbstractShaderProgram: public AbstractObject { /** * @brief Link the shader * - * Returns `false` if linking failed, `true` otherwise. Compiler - * message (if any) is printed to error output. All attached shaders - * must be compiled with @ref Shader::compile() before linking. - * @see @fn_gl{LinkProgram}, @fn_gl{GetProgram} with - * @def_gl{LINK_STATUS} and @def_gl{INFO_LOG_LENGTH}, - * @fn_gl{GetProgramInfoLog} + * Links single shader. If possible, prefer to link multiple shaders + * at once using @ref link(std::initializer_list>) + * for improved performance, see its documentation for more + * information. */ - bool link(); + bool link() { return link({*this}); } /** * @brief Get uniform location diff --git a/src/Magnum/Shader.cpp b/src/Magnum/Shader.cpp index 73ad186e4..64f684379 100644 --- a/src/Magnum/Shader.cpp +++ b/src/Magnum/Shader.cpp @@ -622,57 +622,75 @@ Shader& Shader::addFile(const std::string& filename) { return *this; } -bool Shader::compile() { - CORRADE_ASSERT(_sources.size() > 1, "Shader::compile(): no files added", false); - - /* Array of source string pointers and their lengths */ - /** @todo Use `Containers::ArrayTuple` to avoid one allocation if it ever - gets to be implemented (we need properly aligned memory too) */ - Containers::Array pointers(_sources.size()); - Containers::Array sizes(_sources.size()); - for(std::size_t i = 0; i != _sources.size(); ++i) { - pointers[i] = static_cast(_sources[i].data()); - sizes[i] = _sources[i].size(); +bool Shader::compile(std::initializer_list> shaders) { + bool allSuccess = true; + + /* Allocate large enough array for source pointers and sizes (to avoid + reallocating it for each of them) */ + std::size_t maxSourceCount = 0; + for(Shader& shader: shaders) { + CORRADE_ASSERT(shader._sources.size() > 1, "Shader::compile(): no files added", false); + maxSourceCount = std::max(shader._sources.size(), maxSourceCount); } + Containers::Array pointers(maxSourceCount); + Containers::Array sizes(maxSourceCount); - /* Create shader and set its source */ - glShaderSource(_id, _sources.size(), pointers, sizes); - - /* Compile shader */ - glCompileShader(_id); - - /* Check compilation status */ - GLint success, logLength; - glGetShaderiv(_id, GL_COMPILE_STATUS, &success); - glGetShaderiv(_id, GL_INFO_LOG_LENGTH, &logLength); - - /* Error or warning message. The string is returned null-terminated, scrap - the \0 at the end afterwards */ - std::string message(logLength, '\0'); - if(message.size() > 1) - glGetShaderInfoLog(_id, message.size(), nullptr, &message[0]); - message.resize(std::max(logLength, 1)-1); - - /* Show error log */ - if(!success) { - Error out; - out.setFlag(Debug::NewLineAtTheEnd, false); - out.setFlag(Debug::SpaceAfterEachValue, false); - out << "Shader: " << shaderName(_type) - << " shader failed to compile with the following message:\n" - << message; - - /* Or just message, if any */ - } else if(!message.empty()) { - Error out; - out.setFlag(Debug::NewLineAtTheEnd, false); - out.setFlag(Debug::SpaceAfterEachValue, false); - out << "Shader: " << shaderName(_type) - << " shader was successfully compiled with the following message:\n" - << message; + /* Upload sources of all shaders */ + for(Shader& shader: shaders) { + for(std::size_t i = 0; i != shader._sources.size(); ++i) { + pointers[i] = static_cast(shader._sources[i].data()); + sizes[i] = shader._sources[i].size(); + } + + glShaderSource(shader._id, shader._sources.size(), pointers, sizes); + } + + /* Invoke (possibly parallel) compilation on all shaders */ + for(Shader& shader: shaders) glCompileShader(shader._id); + + /* After compilation phase, check status of all shaders */ + Int i = 1; + for(Shader& shader: shaders) { + GLint success, logLength; + glGetShaderiv(shader._id, GL_COMPILE_STATUS, &success); + glGetShaderiv(shader._id, GL_INFO_LOG_LENGTH, &logLength); + + /* Error or warning message. The string is returned null-terminated, + scrap the \0 at the end afterwards */ + std::string message(logLength, '\0'); + if(message.size() > 1) + glGetShaderInfoLog(shader._id, message.size(), nullptr, &message[0]); + message.resize(std::max(logLength, 1)-1); + + /* Show error log */ + if(!success) { + Error out; + out.setFlag(Debug::NewLineAtTheEnd, false); + out.setFlag(Debug::SpaceAfterEachValue, false); + out << "Shader::compile(): compilation of " << shaderName(shader._type) + << " shader"; + if(shaders.size() != 1) out << ' ' << std::to_string(i); + out << " failed with the following message:\n" + << message; + + /* Or just warnings, if any */ + } else if(!message.empty()) { + Error out; + out.setFlag(Debug::NewLineAtTheEnd, false); + out.setFlag(Debug::SpaceAfterEachValue, false); + out << "Shader::compile(): compilation of " << shaderName(shader._type) + << " shader"; + if(shaders.size() != 1) out << ' ' << std::to_string(i); + out << " succeeded with the following message:\n" + << message; + } + + /* Success of all depends on each of them */ + allSuccess = allSuccess && success; + ++i; } - return success; + return allSuccess; } #ifndef DOXYGEN_GENERATING_OUTPUT diff --git a/src/Magnum/Shader.h b/src/Magnum/Shader.h index e7768948a..d3a077f86 100644 --- a/src/Magnum/Shader.h +++ b/src/Magnum/Shader.h @@ -29,8 +29,9 @@ * @brief Class @ref Magnum::Shader */ -#include +#include #include +#include #include "Magnum/AbstractObject.h" #include "Magnum/Magnum.h" @@ -431,6 +432,20 @@ class MAGNUM_EXPORT Shader: public AbstractObject { static Int maxCombinedUniformComponents(Type type); #endif + /** + * @brief Compile multiple shaders simultaenously + * + * Returns `false` if compilation of any shader failed, `true` if + * everything succeeded. Compiler messages (if any) are printed to + * error output. The operation is batched in a way that allows the + * driver to perform multiple compilations simultaenously (i.e. in + * multiple threads). + * @see @fn_gl{ShaderSource}, @fn_gl{CompileShader}, @fn_gl{GetShader} + * with @def_gl{COMPILE_STATUS} and @def_gl{INFO_LOG_LENGTH}, + * @fn_gl{GetShaderInfoLog} + */ + static bool compile(std::initializer_list> shaders); + /** * @brief Constructor * @param version Target version @@ -524,13 +539,12 @@ class MAGNUM_EXPORT Shader: public AbstractObject { /** * @brief Compile shader * - * Returns `false` if compilation failed, `true` otherwise. Compiler - * message (if any) is printed to error output. - * @see @fn_gl{ShaderSource}, @fn_gl{CompileShader}, @fn_gl{GetShader} - * with @def_gl{COMPILE_STATUS} and @def_gl{INFO_LOG_LENGTH}, - * @fn_gl{GetShaderInfoLog} + * Compiles single shader. Prefer to compile multiple shaders at once + * using @ref compile(std::initializer_list>) + * for improved performance, see its documentation for more + * information. */ - bool compile(); + bool compile() { return compile({*this}); } private: Type _type;