tdf#93614 - detect hanging OpenGL drivers with a watchdog.

If an OpenGL zone takes >2s to make progress, disable OpenGL.
If an OpenGL zone takes >5s to make progress, abort the app.

Reviewed-on: https://gerrit.libreoffice.org/17986Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Michael Meeks <michael.meeks@collabora.com>

Conflicts:
	vcl/workben/vcldemo.cxx

Change-Id: I776c06a3f8ba460ff9842a9130c21f9ee2147eee
Reviewed-on: https://gerrit.libreoffice.org/17995Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Michael Meeks <michael.meeks@collabora.com>

vcl/inc/opengl/watchdog.hxx

+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef INCLUDED_VCL_INC_OPENGL_WATCHDOG_H
+#define INCLUDED_VCL_INC_OPENGL_WATCHDOG_H
+
+#include <sal/config.h>
+#include <sal/types.h>
+#include <rtl/ref.hxx>
+#include <salhelper/thread.hxx>
+
+class OpenGLWatchdogThread : private salhelper::Thread
+{
+    OpenGLWatchdogThread();
+    virtual void execute() SAL_OVERRIDE;
+public:
+    using salhelper::Thread::acquire;
+    using salhelper::Thread::release;
+    static void start();
+    static void stop();
+};
+
+#endif // INCLUDED_VCL_INC_OPENGL_WATCHDOG_H
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

vcl/inc/opengl/zone.hxx

@@ -7,28 +7,39 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-#ifndef INCLUDED_VCL_INC_OPENGL_GUARD_H
-#define INCLUDED_VCL_INC_OPENGL_GUARD_H
+#ifndef INCLUDED_VCL_INC_OPENGL_ZONE_H
+#define INCLUDED_VCL_INC_OPENGL_ZONE_H
 
 #include <sal/config.h>
+#include <sal/types.h>
+#include <vcl/dllapi.h>
+
+class OpenGLZoneTest;
+class OpenGLWatchdogThread;
 
 /**
  * We want to be able to detect if a given crash came
  * from the OpenGL code, so use this helper to track that.
  */
-class OpenGLSalGraphicsImpl;
 class OpenGLZone {
-    static int gnInOpenGLZone;
+    friend class OpenGLZoneTest;
+    friend class OpenGLWatchdogThread;
     friend class OpenGLSalGraphicsImpl;
-    static void enter() { gnInOpenGLZone++; }
-    static void leave() { gnInOpenGLZone--; }
+
+    /// how many times have we entered a GL zone
+    static volatile sal_uInt64 gnEnterCount;
+    /// how many times have we left a new GL zone
+    static volatile sal_uInt64 gnLeaveCount;
+
+    static VCL_DLLPUBLIC void enter();
+    static VCL_DLLPUBLIC void leave();
 public:
-     OpenGLZone() { enter(); }
-    ~OpenGLZone() { leave(); }
-    static bool isInZone() { return gnInOpenGLZone > 0; }
+     OpenGLZone() { gnEnterCount++; }
+    ~OpenGLZone() { gnLeaveCount++; }
+    static bool isInZone() { return gnEnterCount != gnLeaveCount; }
     static void hardDisable();
 };
 
-#endif // INCLUDED_VCL_INC_OPENGL_PROGRAM_H
+#endif // INCLUDED_VCL_INC_OPENGL_ZONE_H
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

vcl/source/app/svmain.cxx

@@ -82,6 +82,7 @@
 #include "uno/current_context.hxx"
 
 #include "opengl/zone.hxx"
+#include "opengl/watchdog.hxx"
 
 #if OSL_DEBUG_LEVEL > 0
 #include <typeinfo>
@@ -189,7 +190,10 @@ int ImplSVMain()
       pSVData->mxAccessBridge.clear();
     }
 
+    OpenGLWatchdogThread::stop();
+
     DeInitVCL();
+
     return nReturn;
 }
 

vcl/source/opengl/OpenGLHelper.cxx

@@ -26,6 +26,8 @@
 #include <vector>
 
 #include "opengl/zone.hxx"
+#include "opengl/watchdog.hxx"
+#include <osl/conditn.h>
 
 #if defined UNX && !defined MACOSX && !defined IOS && !defined ANDROID
 #include "opengl/x11/X11DeviceInfo.hxx"
@@ -446,11 +448,101 @@ bool OpenGLHelper::supportsVCLOpenGL()
         return true;
 }
 
-/// How many nested OpenGL code-paths are we inside ?
-int OpenGLZone::gnInOpenGLZone = 0;
+sal_uInt64 volatile OpenGLZone::gnEnterCount = 0;
+sal_uInt64 volatile OpenGLZone::gnLeaveCount = 0;
+
+void OpenGLZone::enter() { gnEnterCount++; }
+void OpenGLZone::leave() { gnLeaveCount++; }
+
+namespace {
+    static volatile bool gbWatchdogFiring = false;
+    static oslCondition gpWatchdogExit = NULL;
+    static rtl::Reference<OpenGLWatchdogThread> gxWatchdog;
+}
+
+OpenGLWatchdogThread::OpenGLWatchdogThread()
+    : salhelper::Thread("OpenGL Watchdog")
+{
+}
+
+void OpenGLWatchdogThread::execute()
+{
+    static const int nDisableEntries = 4; // 2 seconds - disable GL
+    static const int nAbortAfter = 10;   // 5 seconds - not coming back; abort
+    int nUnchanged = 0; // how many unchanged nEnters
+
+    TimeValue aHalfSecond;
+    aHalfSecond.Seconds = 0;
+    aHalfSecond.Nanosec = 1000*1000*1000/2;
+
+    do {
+        sal_uInt64 nLastEnters = OpenGLZone::gnEnterCount;
+
+        osl_waitCondition(gpWatchdogExit, &aHalfSecond);
+
+        if (OpenGLZone::isInZone())
+        {
+            if (nLastEnters == OpenGLZone::gnEnterCount)
+                nUnchanged++;
+            else
+                nUnchanged = 0;
+            SAL_INFO("vcl.opengl", "GL watchdog - unchanged " <<
+                     nUnchanged << " enter count " <<
+                     OpenGLZone::gnEnterCount);
+
+            // Not making progress
+            if (nUnchanged == nDisableEntries)
+            {
+                gbWatchdogFiring = true;
+                SAL_WARN("vcl.opengl", "Watchdog triggered: hard disable GL");
+                OpenGLZone::hardDisable();
+                gbWatchdogFiring = false;
+            }
+
+            if (nUnchanged == nAbortAfter)
+            {
+                SAL_WARN("vcl.opengl", "Watchdog gave up: aborting");
+                gbWatchdogFiring = true;
+                std::abort();
+            }
+        }
+        else
+        {
+            nUnchanged = 0;
+        }
+    } while (!osl_checkCondition(gpWatchdogExit));
+}
+
+void OpenGLWatchdogThread::start()
+{
+    assert (gxWatchdog == NULL);
+    gpWatchdogExit = osl_createCondition();
+    gxWatchdog = rtl::Reference<OpenGLWatchdogThread>(new OpenGLWatchdogThread());
+    gxWatchdog->launch();
+}
+
+void OpenGLWatchdogThread::stop()
+{
+    if (gbWatchdogFiring)
+        return; // in watchdog thread
+
+    if (gpWatchdogExit)
+        osl_setCondition(gpWatchdogExit);
+
+    if (gxWatchdog.is())
+    {
+        gxWatchdog->join();
+        gxWatchdog.clear();
+    }
+
+    if (gpWatchdogExit)
+        osl_destroyCondition(gpWatchdogExit);
+    gpWatchdogExit = NULL;
+}
 
 /**
- * Called from a signal handler if we get a crash in some GL code
+ * Called from a signal handler or watchdog thread if we get
+ * a crash or hang in some GL code.
  */
 void OpenGLZone::hardDisable()
 {
@@ -471,6 +563,8 @@ void OpenGLZone::hardDisable()
             css::configuration::theDefaultProvider::get(
                 comphelper::getProcessComponentContext()),
             css::uno::UNO_QUERY_THROW)->flush();
+
+        OpenGLWatchdogThread::stop();
     }
 }
 
@@ -502,25 +596,31 @@ bool OpenGLHelper::isVCLOpenGLEnabled()
 
     bSet = true;
     bForceOpenGL = !!getenv("SAL_FORCEGL") || officecfg::Office::Common::VCL::ForceOpenGL::get();
+
+    bool bRet = false;
     if (bForceOpenGL)
-        return true;
+        bRet = true;
 
-    if (!supportsVCLOpenGL())
+    else if (!supportsVCLOpenGL())
+        bRet = false;
+    else
     {
-        return false;
-    }
+        static bool bEnableGLEnv = !!getenv("SAL_ENABLEGL");
 
-    static bool bEnableGLEnv = !!getenv("SAL_ENABLEGL");
+        bEnable = bEnableGLEnv;
 
-    bEnable = bEnableGLEnv;
+        static bool bDuringBuild = getenv("VCL_HIDE_WINDOWS");
+        if (bDuringBuild && !bEnable /* env. enable overrides */)
+            bEnable = false;
+        else if (officecfg::Office::Common::VCL::UseOpenGL::get())
+            bEnable = true;
 
-    static bool bDuringBuild = getenv("VCL_HIDE_WINDOWS");
-    if (bDuringBuild && !bEnable /* env. enable overrides */)
-        bEnable = false;
-    else if (officecfg::Office::Common::VCL::UseOpenGL::get())
-        bEnable = true;
+        bRet = bEnable;
+    }
+    if (bRet)
+        OpenGLWatchdogThread::start();
 
-    return bEnable;
+    return bRet;
 }
 
 #if defined UNX && !defined MACOSX && !defined IOS && !defined ANDROID && !defined(LIBO_HEADLESS)

vcl/workben/vcldemo.cxx

@@ -41,6 +41,7 @@
 #include <basegfx/numeric/ftools.hxx>
 #include <basegfx/matrix/b2dhommatrix.hxx>
 #include <vcldemo-debug.hxx>
+#include <opengl/zone.hxx>
 
 #include <rtl/math.hxx>
 
@@ -1457,13 +1458,23 @@ class DemoWidgets : public WorkWindow
     VclPtr<VclBox> mpBox;
     VclPtr<ToolBox> mpToolbox;
     VclPtr<PushButton> mpButton;
+    VclPtr<VclHBox> mpHBox;
+    VclPtr<CheckBox> mpGLCheck;
+    VclPtr<ComboBox> mpGLCombo;
+    VclPtr<PushButton> mpGLButton;
+
+    DECL_LINK(GLTestClick, void *);
 
 public:
     DemoWidgets() :
         WorkWindow(NULL, WB_STDWORK),
         mpBox(VclPtrInstance<VclVBox>(this, false, 3)),
         mpToolbox(VclPtrInstance<ToolBox>(mpBox.get())),
-        mpButton(VclPtrInstance<PushButton>(mpBox.get()))
+        mpButton(VclPtrInstance<PushButton>(mpBox.get())),
+        mpHBox(VclPtrInstance<VclHBox>(mpBox.get(), true, 3)),
+        mpGLCheck(VclPtrInstance<CheckBox>(mpHBox.get())),
+        mpGLCombo(VclPtrInstance<ComboBox>(mpHBox.get())),
+        mpGLButton(VclPtrInstance<PushButton>(mpHBox.get()))
     {
         SetText("VCL widget demo");
 
@@ -1484,11 +1495,27 @@ public:
         mpButton->SetText("Click me; go on");
         mpButton->Show();
 
+        mpGLCheck->SetText("Test in OGL zone");
+        mpGLCheck->Show();
+        mpGLCombo->InsertEntry("sleep 1 second");
+        mpGLCombo->InsertEntry("sleep 3 seconds");
+        mpGLCombo->InsertEntry("sleep 7 seconds");
+        mpGLCombo->SelectEntryPos(2);
+        mpGLCombo->Show();
+        mpGLButton->SetText("Execute test");
+        mpGLButton->SetClickHdl(LINK(this,DemoWidgets,GLTestClick));
+        mpGLButton->Show();
+        mpHBox->Show();
+
         Show();
     }
     virtual ~DemoWidgets() { disposeOnce(); }
     virtual void dispose() SAL_OVERRIDE
     {
+        mpGLButton.disposeAndClear();
+        mpGLCombo.disposeAndClear();
+        mpGLCheck.disposeAndClear();
+        mpHBox.disposeAndClear();
         mpBox.disposeAndClear();
         mpToolbox.disposeAndClear();
         mpButton.disposeAndClear();
@@ -1521,6 +1548,46 @@ public:
     }
 };
 
+class OpenGLZoneTest {
+public:
+    static void enter() { OpenGLZone::enter(); }
+    static void leave() { OpenGLZone::leave(); }
+};
+
+IMPL_LINK_NOARG(DemoWidgets,GLTestClick)
+{
+    sal_Int32 nSelected = mpGLCombo->GetSelectEntryPos();
+
+    TimeValue aDelay;
+    aDelay.Seconds = 0;
+    aDelay.Nanosec = 0;
+    switch (nSelected)
+    {
+    case 0:
+        aDelay.Seconds = 1;
+        break;
+    case 1:
+        aDelay.Seconds = 3;
+        break;
+    case 2:
+        aDelay.Seconds = 7;
+        break;
+    default:
+        break;
+    }
+
+    bool bEnterLeave = mpGLCheck->IsChecked();
+    if (bEnterLeave)
+        OpenGLZoneTest::enter();
+
+    osl_waitThread(&aDelay);
+
+    if (bEnterLeave)
+        OpenGLZoneTest::leave();
+
+    return 0;
+}
+
 class DemoPopup : public FloatingWindow
 {
  public: