tdf#93814: Added support for caching shader program binaries.

Reviewed-on: https://gerrit.libreoffice.org/18555Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Tomaž Vajngerl <quikee@gmail.com>
Signed-off-by: Michael Meeks <michael.meeks@collabora.com>

Conflicts:
	include/vcl/opengl/OpenGLContext.hxx
	include/vcl/opengl/OpenGLHelper.hxx
	vcl/inc/opengl/win/WinDeviceInfo.hxx
	vcl/source/opengl/OpenGLContext.cxx

Change-Id: I21c844b47282f6b3eec443933a86421a074e24df

include/vcl/opengl/OpenGLContext.hxx

@@ -56,9 +56,13 @@ class NSOpenGLView;
 #include <vcl/window.hxx>
 #include <tools/gen.hxx>
 #include <vcl/syschild.hxx>
+#include <rtl/crc.h>
 #include <rtl/ref.hxx>
 
+#include <map>
+#include <memory>
 #include <set>
+#include <unordered_map>
 
 class OpenGLFramebuffer;
 class OpenGLProgram;
@@ -276,15 +280,15 @@ private:
     OpenGLFramebuffer* mpFirstFramebuffer;
     OpenGLFramebuffer* mpLastFramebuffer;
 
-    struct ProgramKey
+    struct ProgramHash
     {
-        ProgramKey( const OUString& vertexShader, const OUString& fragmentShader, const OString& preamble );
-        bool operator< ( const ProgramKey& other ) const;
-        OUString vertexShader;
-        OUString fragmentShader;
-        OString preamble;
+        size_t operator()( const rtl::OString& aDigest ) const
+        {
+            return (size_t)( rtl_crc32( 0, aDigest.getStr(), aDigest.getLength() ) );
+        }
     };
-    std::map<ProgramKey, boost::shared_ptr<OpenGLProgram> > maPrograms;
+    typedef std::unordered_map< rtl::OString, std::shared_ptr<OpenGLProgram>, ProgramHash > ProgramCollection;
+    ProgramCollection maPrograms;
     OpenGLProgram* mpCurrentProgram;
 #ifdef DBG_UTIL
     std::set<SalGraphicsImpl*> maParents;

include/vcl/opengl/OpenGLHelper.hxx

@@ -36,8 +36,13 @@
 
 class VCL_DLLPUBLIC OpenGLHelper
 {
+    OpenGLHelper() SAL_DELETED_FUNCTION; // Should not be instantiated
+
 public:
-    static GLint LoadShaders(const OUString& rVertexShaderName, const OUString& rFragmentShaderName, const OString& preamble = "" );
+
+    static rtl::OString GetDigest(const OUString& rVertexShaderName, const OUString& rFragmentShaderName, const rtl::OString& preamble = "" );
+
+    static GLint LoadShaders(const OUString& rVertexShaderName, const OUString& rFragmentShaderName, const rtl::OString& preamble = "", const rtl::OString& rDigest = "" );
 
     /**
      * The caller is responsible for allocate the memory for the RGBA buffer, before call

vcl/inc/opengl/program.hxx

@@ -50,7 +50,8 @@ public:
     OpenGLProgram();
     ~OpenGLProgram();
 
-    bool Load( const OUString& rVertexShader, const OUString& rFragmentShader, const OString& preamble = "" );
+    bool Load( const OUString& rVertexShader, const OUString& rFragmentShader,
+               const rtl::OString& preamble = "", const rtl::OString& rDigest = "" );
     bool Use();
     bool Clean();
 

vcl/inc/opengl/win/WinDeviceInfo.hxx

@@ -75,15 +75,8 @@ enum DeviceVendor {
 
 struct DriverInfo
 {
-    typedef std::vector<OUString> DeviceFamilyVector;
-
-    // If |ownDevices| is true, you are transferring ownership of the devices
-    // array, and it will be deleted when this GfxDriverInfo is destroyed.
-
-    DriverInfo(OperatingSystem os, const OUString& vendor, DeviceFamilyVector* devices,
-            VersionComparisonOp op,
-            uint64_t driverVersion, bool bWhiteListed = false, const char *suggestedVersion = nullptr,
-            bool ownDevices = false);
+    DriverInfo(OperatingSystem os, const OUString& vendor, VersionComparisonOp op,
+            uint64_t driverVersion, bool bWhiteListed = false, const char *suggestedVersion = nullptr);
 
     DriverInfo();
     DriverInfo(const DriverInfo&);
@@ -183,6 +176,51 @@ public:
     virtual ~WinOpenGLDeviceInfo();
 
     virtual bool isDeviceBlocked();
+
+    const OUString& GetDriverVersion() const
+    {
+        return maDriverVersion;
+    }
+
+    const OUString& GetDriverDate() const
+    {
+        return maDriverDate;
+    }
+
+    const OUString& GetDeviceID() const
+    {
+        return maDeviceID;
+    }
+
+    const OUString& GetAdapterVendorID() const
+    {
+        return maAdapterVendorID;
+    }
+
+    const OUString& GetAdapterDeviceID() const
+    {
+        return maAdapterDeviceID;
+    }
+
+    const OUString& GetAdapterSubsysID() const
+    {
+        return maAdapterSubsysID;
+    }
+    const OUString& GetDeviceKey() const
+    {
+        return maDeviceKey;
+    }
+
+    const OUString& GetDeviceString() const
+    {
+        return maDeviceString;
+    }
+
+    sal_uInt32 GetWindowsVersion() const
+    {
+        return mnWindowsVersion;
+    }
+
 };
 
 #endif

vcl/inc/opengl/x11/X11DeviceInfo.hxx

@@ -44,6 +44,32 @@ public:
     virtual ~X11OpenGLDeviceInfo();
 
     virtual bool isDeviceBlocked() SAL_OVERRIDE;
+
+    const OString& GetVendor() const
+    {
+        return maVendor;
+    }
+
+    const OString& GetRenderer() const
+    {
+        return maRenderer;
+    }
+
+    const OString& GetVersion() const
+    {
+        return maVersion;
+    }
+
+    const OString& GetOS() const
+    {
+        return maOS;
+    }
+
+    const OString& GetOSRelease() const
+    {
+        return maOSRelease;
+    }
+
 };
 
 #endif

vcl/opengl/program.cxx

@@ -36,9 +36,12 @@ OpenGLProgram::~OpenGLProgram()
         glDeleteProgram( mnId );
 }
 
-bool OpenGLProgram::Load( const OUString& rVertexShader, const OUString& rFragmentShader, const OString& preamble )
+bool OpenGLProgram::Load( const OUString& rVertexShader,
+                          const OUString& rFragmentShader,
+                          const rtl::OString& preamble,
+                          const rtl::OString& rDigest )
 {
-    mnId = OpenGLHelper::LoadShaders( rVertexShader, rFragmentShader, preamble );
+    mnId = OpenGLHelper::LoadShaders( rVertexShader, rFragmentShader, preamble, rDigest );
     return ( mnId != 0 );
 }
 

vcl/source/opengl/OpenGLContext.cxx

@@ -1725,22 +1725,25 @@ void OpenGLContext::ReleaseFramebuffers()
     BindFramebuffer( NULL );
 }
 
-OpenGLProgram* OpenGLContext::GetProgram( const OUString& rVertexShader, const OUString& rFragmentShader, const OString& preamble )
+OpenGLProgram* OpenGLContext::GetProgram( const OUString& rVertexShader, const OUString& rFragmentShader, const rtl::OString& preamble )
 {
     OpenGLZone aZone;
 
-    ProgramKey aKey( rVertexShader, rFragmentShader, preamble );
+    rtl::OString aKey = OpenGLHelper::GetDigest( rVertexShader, rFragmentShader, preamble );
 
-    std::map< ProgramKey, boost::shared_ptr<OpenGLProgram> >::iterator
-        it = maPrograms.find( aKey );
-    if( it != maPrograms.end() )
-        return it->second.get();
+    if( !aKey.isEmpty() )
+    {
+        ProgramCollection::iterator it = maPrograms.find( aKey );
+        if( it != maPrograms.end() )
+            return it->second.get();
+    }
 
-    boost::shared_ptr<OpenGLProgram> pProgram = boost::make_shared<OpenGLProgram>();
-    if( !pProgram->Load( rVertexShader, rFragmentShader, preamble ) )
+    std::shared_ptr<OpenGLProgram> pProgram = std::make_shared<OpenGLProgram>();
+    if( !pProgram->Load( rVertexShader, rFragmentShader, preamble, aKey ) )
         return NULL;
 
-    maPrograms.insert(std::pair<ProgramKey, boost::shared_ptr<OpenGLProgram> >(aKey, pProgram));
+    maPrograms.insert(std::make_pair(aKey, pProgram));
+
     return pProgram.get();
 }
 
@@ -1766,20 +1769,4 @@ OpenGLProgram* OpenGLContext::UseProgram( const OUString& rVertexShader, const O
     return mpCurrentProgram;
 }
 
-inline
-OpenGLContext::ProgramKey::ProgramKey( const OUString& v, const OUString& f, const OString& p )
-: vertexShader( v ), fragmentShader( f ), preamble( p )
-{
-}
-
-inline
-bool OpenGLContext::ProgramKey::operator< ( const ProgramKey& other ) const
-{
-    if( vertexShader != other.vertexShader )
-        return vertexShader < other.vertexShader;
-    if( fragmentShader != other.fragmentShader )
-        return fragmentShader < other.fragmentShader;
-    return preamble < other.preamble;
-}
-
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

vcl/source/opengl/OpenGLHelper.cxx

@@ -12,6 +12,9 @@
 
 #include <osl/file.hxx>
 #include <rtl/bootstrap.hxx>
+#include <rtl/digest.h>
+#include <rtl/strbuf.hxx>
+#include <rtl/ustring.hxx>
 #include <config_folders.h>
 #include <vcl/salbtype.hxx>
 #include <vcl/bmpacc.hxx>
@@ -25,6 +28,8 @@
 
 #include <stdarg.h>
 #include <vector>
+#include <deque>
+#include <unordered_map>
 
 #include "svdata.hxx"
 #include "salgdi.hxx"
@@ -47,6 +52,8 @@ sal_uInt64 volatile OpenGLZone::gnLeaveCount = 0;
 
 namespace {
 
+using namespace rtl;
+
 OUString getShaderFolder()
 {
     OUString aUrl("$BRAND_BASE_DIR/" LIBO_ETC_FOLDER);
@@ -143,12 +150,247 @@ static void addPreamble(OString& rShaderSource, const OString& rPreamble)
     }
 }
 
-GLint OpenGLHelper::LoadShaders(const OUString& rVertexShaderName,const OUString& rFragmentShaderName, const OString& preamble)
+namespace
+{
+    static const sal_uInt32 GLenumSize = sizeof(GLenum);
+
+    OString getHexString(const sal_uInt8* pData, sal_uInt32 nLength)
+    {
+        static const char* pHexData = "0123456789ABCDEF";
+
+        bool bIsZero = true;
+        OStringBuffer aHexStr;
+        for(size_t i = 0; i < nLength; ++i)
+        {
+            sal_uInt8 val = pData[i];
+            if( val != 0 )
+                bIsZero = false;
+            aHexStr.append( pHexData[ val & 0xf ] );
+            aHexStr.append( pHexData[ val >> 4 ] );
+        }
+        if( bIsZero )
+            return OString();
+        else
+            return aHexStr.makeStringAndClear();
+    }
+
+    OString generateMD5(const void* pData, size_t length)
+    {
+        sal_uInt8 pBuffer[RTL_DIGEST_LENGTH_MD5];
+        rtlDigestError aError = rtl_digest_MD5(pData, length,
+                pBuffer, RTL_DIGEST_LENGTH_MD5);
+        SAL_WARN_IF(aError != rtl_Digest_E_None, "vcl.opengl", "md5 generation failed");
+
+        return getHexString(pBuffer, RTL_DIGEST_LENGTH_MD5);
+    }
+
+    OString getStringDigest( const OUString& rVertexShaderName,
+                             const OUString& rFragmentShaderName,
+                             const OString& rPreamble )
+    {
+        // read shaders source
+        OString aVertexShaderSource = loadShader( rVertexShaderName );
+        OString aFragmentShaderSource = loadShader( rFragmentShaderName );
+
+        // get info about the graphic device
+#if defined( SAL_UNX ) && !defined( MACOSX ) && !defined( IOS )&& !defined( ANDROID )
+        static const X11OpenGLDeviceInfo aInfo;
+        static const OString aDeviceInfo (
+                aInfo.GetOS() +
+                aInfo.GetOSRelease() +
+                aInfo.GetRenderer() +
+                aInfo.GetVendor() +
+                aInfo.GetVersion() );
+#elif defined( _WIN32 )
+        static const WinOpenGLDeviceInfo aInfo;
+        static const OString aDeviceInfo (
+                OUStringToOString( aInfo.GetAdapterVendorID(), RTL_TEXTENCODING_UTF8 ) +
+                OUStringToOString( aInfo.GetAdapterDeviceID(), RTL_TEXTENCODING_UTF8 ) +
+                OUStringToOString( aInfo.GetDriverVersion(), RTL_TEXTENCODING_UTF8 ) +
+                OString::number( aInfo.GetWindowsVersion() ) );
+#else
+        static const OString aDeviceInfo (
+                OString( (const char*)(glGetString(GL_VENDOR)) ) +
+                OString( (const char*)(glGetString(GL_RENDERER)) ) +
+                OString( (const char*)(glGetString(GL_VERSION)) ) );
+#endif
+
+        OString aMessage;
+        aMessage += rPreamble;
+        aMessage += aVertexShaderSource;
+        aMessage += aFragmentShaderSource;
+        aMessage += aDeviceInfo;
+
+        return generateMD5(aMessage.getStr(), aMessage.getLength());
+    }
+
+    OString getCacheFolder()
+    {
+        OUString url("${$BRAND_BASE_DIR/" LIBO_ETC_FOLDER "/" SAL_CONFIGFILE("bootstrap") ":UserInstallation}/cache/");
+        rtl::Bootstrap::expandMacros(url);
+
+        osl::Directory::create(url);
+
+        return rtl::OUStringToOString(url, RTL_TEXTENCODING_UTF8);
+    }
+
+
+    bool writeProgramBinary( const OString& rBinaryFileName,
+                             const std::vector<sal_uInt8>& rBinary )
+    {
+        osl::File aFile(rtl::OStringToOUString(rBinaryFileName, RTL_TEXTENCODING_UTF8));
+        osl::FileBase::RC eStatus = aFile.open(
+                osl_File_OpenFlag_Write | osl_File_OpenFlag_Create );
+
+        if( eStatus != osl::FileBase::E_None )
+        {
+            // when file already exists we do not have to save it:
+            // we can be sure that the binary to save is exactly equal
+            // to the already saved binary, since they have the same hash value
+            if( eStatus == osl::FileBase::E_EXIST )
+            {
+                SAL_WARN( "vcl.opengl",
+                        "No binary program saved. A file with the same hash already exists: '" << rBinaryFileName << "'" );
+                return true;
+            }
+            return false;
+        }
+
+        sal_uInt64 nBytesWritten = 0;
+        aFile.write( rBinary.data(), rBinary.size(), nBytesWritten );
+
+        assert( rBinary.size() == nBytesWritten );
+
+        return true;
+    }
+
+    bool readProgramBinary( const OString& rBinaryFileName,
+                            std::vector<sal_uInt8>& rBinary )
+    {
+        osl::File aFile( rtl::OStringToOUString( rBinaryFileName, RTL_TEXTENCODING_UTF8 ) );
+        if(aFile.open( osl_File_OpenFlag_Read ) == osl::FileBase::E_None)
+        {
+            sal_uInt64 nSize = 0;
+            aFile.getSize( nSize );
+            rBinary.resize( nSize );
+            sal_uInt64 nBytesRead = 0;
+            aFile.read( rBinary.data(), nSize, nBytesRead );
+            assert( nSize == nBytesRead );
+            SAL_WARN("vcl.opengl", "Loading file: '" << rBinaryFileName << "': success" );
+            return true;
+        }
+        else
+        {
+            SAL_WARN("vcl.opengl", "Loading file: '" << rBinaryFileName << "': FAIL");
+        }
+
+        return false;
+    }
+
+    OString createFileName( const OUString& rVertexShaderName,
+                            const OUString& rFragmentShaderName,
+                            const OString& rDigest )
+    {
+        OString aFileName;
+        aFileName += getCacheFolder();
+        aFileName += rtl::OUStringToOString( rVertexShaderName, RTL_TEXTENCODING_UTF8 ) + "-";
+        aFileName += rtl::OUStringToOString( rFragmentShaderName, RTL_TEXTENCODING_UTF8 ) + "-";
+        aFileName += rDigest + ".bin";
+        return aFileName;
+    }
+
+    GLint loadProgramBinary( GLuint nProgramID, const OString& rBinaryFileName )
+    {
+        GLint nResult = GL_FALSE;
+        GLenum nBinaryFormat;
+        std::vector<sal_uInt8> aBinary;
+        if( readProgramBinary( rBinaryFileName, aBinary ) && aBinary.size() > GLenumSize )
+        {
+            GLint nBinaryLength = aBinary.size() - GLenumSize;
+
+            // Extract binary format
+            sal_uInt8* pBF = (sal_uInt8*)(&nBinaryFormat);
+            for( size_t i = 0; i < GLenumSize; ++i )
+            {
+                pBF[i] = aBinary[nBinaryLength + i];
+            }
+
+            // Load the program
+            glProgramBinary( nProgramID, nBinaryFormat, (void*)(aBinary.data()), nBinaryLength );
+
+            // Check the program
+            glGetProgramiv(nProgramID, GL_LINK_STATUS, &nResult);
+        }
+        return nResult;
+    }
+
+    void saveProgramBinary( GLint nProgramID, const OString& rBinaryFileName )
+    {
+        GLint nBinaryLength = 0;
+        GLenum nBinaryFormat = GL_NONE;
+
+        glGetProgramiv( nProgramID, GL_PROGRAM_BINARY_LENGTH, &nBinaryLength );
+        if( !( nBinaryLength > 0 ) )
+        {
+            SAL_WARN( "vcl.opengl", "Binary size is zero" );
+            return;
+        }
+
+        std::vector<sal_uInt8> aBinary( nBinaryLength + GLenumSize );
+
+        glGetProgramBinary( nProgramID, nBinaryLength, NULL, &nBinaryFormat, (void*)(aBinary.data()) );
+
+        const sal_uInt8* pBF = (const sal_uInt8*)(&nBinaryFormat);
+        aBinary.insert( aBinary.end(), pBF, pBF + GLenumSize );
+
+        SAL_INFO("vcl.opengl", "Program id: " << nProgramID );
+        SAL_INFO("vcl.opengl", "Binary length: " << nBinaryLength );
+        SAL_INFO("vcl.opengl", "Binary format: " << nBinaryFormat );
+
+        if( !writeProgramBinary( rBinaryFileName, aBinary ) )
+            SAL_WARN("vcl.opengl", "Writing binary file '" << rBinaryFileName << "': FAIL");
+        else
+            SAL_WARN("vcl.opengl", "Writing binary file '" << rBinaryFileName << "': success");
+    }
+}
+
+rtl::OString OpenGLHelper::GetDigest( const OUString& rVertexShaderName,
+                                      const OUString& rFragmentShaderName,
+                                      const OString& rPreamble )
+{
+    return getStringDigest(rVertexShaderName, rFragmentShaderName, rPreamble);
+}
+
+GLint OpenGLHelper::LoadShaders(const OUString& rVertexShaderName,
+                                const OUString& rFragmentShaderName,
+                                const OString& preamble,
+                                const OString& rDigest)
 {
     OpenGLZone aZone;
 
     gbInShaderCompile = true;
 
+    // create the program object
+    GLint ProgramID = glCreateProgram();
+
+    // read shaders from file
+    OString aVertexShaderSource = loadShader(rVertexShaderName);
+    OString aFragmentShaderSource = loadShader(rFragmentShaderName);
+
+
+    GLint bBinaryResult = GL_FALSE;
+    if( GLEW_ARB_get_program_binary && !rDigest.isEmpty() )
+    {
+        OString aFileName =
+                createFileName(rVertexShaderName, rFragmentShaderName, rDigest);
+        bBinaryResult = loadProgramBinary(ProgramID, aFileName);
+        VCL_GL_INFO("vcl.opengl", "Load binary shader from '" << aFileName << "'" << bBinaryResult);
+        CHECK_GL_ERROR();
+    }
+
+    if( bBinaryResult != GL_FALSE )
+        return ProgramID;
+
     VCL_GL_INFO("vcl.opengl", "Load shader: vertex " << rVertexShaderName << " fragment " << rFragmentShaderName);
     // Create the shaders
     GLuint VertexShaderID = glCreateShader(GL_VERTEX_SHADER);
@@ -157,7 +399,6 @@ GLint OpenGLHelper::LoadShaders(const OUString& rVertexShaderName,const OUString
     GLint Result = GL_FALSE;
 
     // Compile Vertex Shader
-    OString aVertexShaderSource = loadShader(rVertexShaderName);
     if( !preamble.isEmpty())
         addPreamble( aVertexShaderSource, preamble );
     char const * VertexSourcePointer = aVertexShaderSource.getStr();
@@ -171,7 +412,6 @@ GLint OpenGLHelper::LoadShaders(const OUString& rVertexShaderName,const OUString
                                 rVertexShaderName, true);
 
     // Compile Fragment Shader
-    OString aFragmentShaderSource = loadShader(rFragmentShaderName);
     if( !preamble.isEmpty())
         addPreamble( aFragmentShaderSource, preamble );
     char const * FragmentSourcePointer = aFragmentShaderSource.getStr();
@@ -185,10 +425,27 @@ GLint OpenGLHelper::LoadShaders(const OUString& rVertexShaderName,const OUString
                                 rFragmentShaderName, true);
 
     // Link the program
-    GLint ProgramID = glCreateProgram();
     glAttachShader(ProgramID, VertexShaderID);
     glAttachShader(ProgramID, FragmentShaderID);
-    glLinkProgram(ProgramID);
+
+    if( GLEW_ARB_get_program_binary && !rDigest.isEmpty() )
+    {
+        glProgramParameteri(ProgramID, GL_PROGRAM_BINARY_RETRIEVABLE_HINT, GL_TRUE);
+        glLinkProgram(ProgramID);
+        glGetProgramiv(ProgramID, GL_LINK_STATUS, &Result);
+        if (!Result)
+        {
+            SAL_WARN("vcl.opengl", "linking failed: " << Result );
+            return LogCompilerError(ProgramID, "program", "<both>", false);
+        }
+        OString aFileName =
+                createFileName(rVertexShaderName, rFragmentShaderName, rDigest);
+        saveProgramBinary(ProgramID, aFileName);
+    }
+    else
+    {
+        glLinkProgram(ProgramID);
+    }
 
     glDeleteShader(VertexShaderID);
     glDeleteShader(FragmentShaderID);