ofz#4052 limit listener range to actually available sheets

... instead of an arbitrary reference range read from a binary
file format, here 4k sheets resulting in >3GB allocated listener
memory.

Change-Id: I629297f4249fdf16a0ede098b63d9648fda69ac3

sc/inc/document.hxx

@@ -2046,6 +2046,24 @@ private:
 
     static ScRecursionHelper*   CreateRecursionHelperInstance();
 
+    /** Adjust a range to available sheets.
+
+        Used to start and stop listening on a sane range. Both o_rRange and
+        o_bEntirelyOutOfBounds are set only if needed and don't have to be
+        initialized by the caller.
+
+        @param  o_bEntirelyOutOfBounds
+                <TRUE/> if both sheets in the range point outside the
+                available sheet range, in which case no adjustment is done and
+                o_rRange is not modified.
+
+        @return <TRUE/> if any adjustment was done or o_bEntirelyOutOfBounds
+                was set <TRUE/>.
+                <FALSE/> if rRange was within the available sheets.
+     */
+    bool                LimitRangeToAvailableSheets( const ScRange& rRange, ScRange& o_rRange,
+                                                     bool& o_bEntirelyOutOfBounds ) const;
+
 public:
     void StartListeningArea( const ScRange& rRange, bool bGroupListening, SvtListener* pListener );
 

sc/source/core/data/documen7.cxx

@@ -44,14 +44,81 @@
 void ScDocument::StartListeningArea(
     const ScRange& rRange, bool bGroupListening, SvtListener* pListener )
 {
-    if ( pBASM )
+    if (!pBASM)
+        return;
+
+    // Ensure sane ranges for the slots, specifically don't attempt to listen
+    // to more sheets than the document has. The slot machine handles it but
+    // with memory waste. Binary import filters can set out-of-bounds ranges
+    // in formula expressions' references, so all middle layers would have to
+    // check it, rather have this central point here.
+    ScRange aLimitedRange( ScAddress::UNINITIALIZED );
+    bool bEntirelyOut;
+    if (!LimitRangeToAvailableSheets( rRange, aLimitedRange, bEntirelyOut))
+    {
         pBASM->StartListeningArea(rRange, bGroupListening, pListener);
+        return;
+    }
+
+    // If both sheets are out-of-bounds in the same direction then just bail out.
+    if (bEntirelyOut)
+        return;
+
+    pBASM->StartListeningArea( aLimitedRange, bGroupListening, pListener);
 }
 
 void ScDocument::EndListeningArea( const ScRange& rRange, bool bGroupListening, SvtListener* pListener )
 {
-    if ( pBASM )
+    if (!pBASM)
+        return;
+
+    // End listening has to limit the range exactly the same as in
+    // StartListeningArea(), otherwise the range would not be found.
+    ScRange aLimitedRange( ScAddress::UNINITIALIZED );
+    bool bEntirelyOut;
+    if (!LimitRangeToAvailableSheets( rRange, aLimitedRange, bEntirelyOut))
+    {
         pBASM->EndListeningArea(rRange, bGroupListening, pListener);
+        return;
+    }
+
+    // If both sheets are out-of-bounds in the same direction then just bail out.
+    if (bEntirelyOut)
+        return;
+
+    pBASM->EndListeningArea( aLimitedRange, bGroupListening, pListener);
+}
+
+bool ScDocument::LimitRangeToAvailableSheets( const ScRange& rRange, ScRange& o_rRange,
+        bool& o_bEntirelyOutOfBounds ) const
+{
+    if (rRange == BCA_LISTEN_ALWAYS)
+        return false;
+
+    const SCTAB nMaxTab = GetTableCount() - 1;
+    if (ValidTab( rRange.aStart.Tab(), nMaxTab) && ValidTab( rRange.aEnd.Tab(), nMaxTab))
+        return false;
+
+    SCTAB nTab1 = rRange.aStart.Tab();
+    SCTAB nTab2 = rRange.aEnd.Tab();
+    SAL_WARN("sc.core","ScDocument::LimitRangeToAvailableSheets - bad sheet range: " << nTab1 << ".." << nTab2 <<
+            ", sheets: 0.." << nMaxTab);
+
+    // Both sheets are out-of-bounds in the same direction.
+    if ((nTab1 < 0 && nTab2 < 0) || (nMaxTab < nTab1 && nMaxTab < nTab2))
+    {
+        o_bEntirelyOutOfBounds = true;
+        return true;
+    }
+
+    // Limit the sheet range to bounds.
+    o_bEntirelyOutOfBounds = false;
+    nTab1 = std::max<SCTAB>( 0, std::min( nMaxTab, nTab1));
+    nTab2 = std::max<SCTAB>( 0, std::min( nMaxTab, nTab2));
+    o_rRange = rRange;
+    o_rRange.aStart.SetTab(nTab1);
+    o_rRange.aEnd.SetTab(nTab2);
+    return true;
 }
 
 void ScDocument::Broadcast( const ScHint& rHint )