fdo#35943: Better performance with pivot table refresh.

When the source data range contains trailing rows, simply skip them, and
don't even bother iterating them.  Apparently sometimes users specify a
data range with a huge amount of trailing empty rows, which would slow
down the pivot table refresh for no good reason.

But we do still need to keep the original end row position, in case the
pivot table needs to include empty cells in the output.

Change-Id: I2c73c368837b8e322e12b25ddf31429488961f06

sc/inc/dpcache.hxx

@@ -121,6 +121,7 @@ private:
     LabelsType maLabelNames;    // Stores dimension names.
     mdds::flat_segment_tree<SCROW, bool> maEmptyRows;
     SCROW mnDataSize;
+    SCROW mnRowCount;
 
     bool mbDisposing;
 

sc/source/core/data/dpcache.cxx

@@ -56,6 +56,7 @@ ScDPCache::ScDPCache(ScDocument* pDoc) :
     mnColumnCount ( 0 ),
     maEmptyRows(0, MAXROW, true),
     mnDataSize(-1),
+    mnRowCount(0),
     mbDisposing(false)
 {
 }
@@ -311,6 +312,16 @@ bool ScDPCache::InitFromDoc(ScDocument* pDoc, const ScRange& rRange)
 
     mnColumnCount = nEndCol - nStartCol + 1;
 
+    // this row count must include the trailing empty rows.
+    mnRowCount = nEndRow - nStartRow; // skip the topmost label row.
+
+    // Skip trailing empty rows if exists.
+    SCCOL nCol1 = nStartCol, nCol2 = nEndCol;
+    SCROW nRow1 = nStartRow, nRow2 = nEndRow;
+    pDoc->ShrinkToDataArea(nDocTab, nCol1, nRow1, nCol2, nRow2);
+    bool bTailEmptyRows = nEndRow > nRow2; // Trailing empty rows exist.
+    nEndRow = nRow2;
+
     maFields.reserve(mnColumnCount);
     for (size_t i = 0; i < static_cast<size_t>(mnColumnCount); ++i)
         maFields.push_back(new Field);
@@ -342,6 +353,17 @@ bool ScDPCache::InitFromDoc(ScDocument* pDoc, const ScRange& rRange)
         }
 
         processBuckets(aBuckets, rField);
+
+        if (bTailEmptyRows)
+        {
+            // If the last item is not empty, append one. Note that the items
+            // are sorted, and empty item should come last when sorted.
+            if (rField.maItems.empty() || !rField.maItems.back().IsEmpty())
+            {
+                aData.SetEmpty();
+                rField.maItems.push_back(aData);
+            }
+        }
     }
 
     PostInit();
@@ -404,6 +426,9 @@ bool ScDPCache::InitFromDataBase(DBConnector& rDB)
 
         rDB.finish();
 
+        if (!maFields.empty())
+            mnRowCount = maFields[0].maData.size();
+
         PostInit();
         return true;
     }
@@ -684,6 +709,8 @@ void ScDPCache::PostInit()
 
 void ScDPCache::Clear()
 {
+    mnColumnCount = 0;
+    mnRowCount = 0;
     maFields.clear();
     maLabelNames.clear();
     maGroupFields.clear();
@@ -723,7 +750,18 @@ SCROW ScDPCache::GetItemDataId(sal_uInt16 nDim, SCROW nRow, bool bRepeatIfEmpty)
     OSL_ENSURE(nDim < mnColumnCount, "ScDPTableDataCache::GetItemDataId ");
 
     const Field& rField = maFields[nDim];
-    if (bRepeatIfEmpty)
+    if (static_cast<size_t>(nRow) >= rField.maData.size())
+    {
+        // nRow is in the trailing empty rows area.
+        if (bRepeatIfEmpty)
+            nRow = rField.maData.size()-1; // Move to the last non-empty row.
+        else
+            // Return the last item, which should always be empty if the
+            // initialization has skipped trailing empty rows.
+            return rField.maItems.size()-1;
+
+    }
+    else if (bRepeatIfEmpty)
     {
         while (nRow > 0 && rField.maItems[rField.maData[nRow]].IsEmpty())
             --nRow;
@@ -772,10 +810,7 @@ const ScDPItemData* ScDPCache::GetItemDataById(long nDim, SCROW nId) const
 
 SCROW ScDPCache::GetRowCount() const
 {
-    if (maFields.empty() || maFields[0].maData.empty())
-        return 0;
-
-    return maFields[0].maData.size();
+    return mnRowCount;
 }
 
 SCROW ScDPCache::GetDataSize() const

sc/source/core/data/dpitemdata.cxx

@@ -33,7 +33,8 @@ sal_Int32 ScDPItemData::Compare(const ScDPItemData& rA, const ScDPItemData& rB)
 {
     if (rA.meType != rB.meType)
     {
-        // group value, value and string in this order.
+        // group value, value and string in this order. Ensure that the empty
+        // type comes last.
         return rA.meType < rB.meType ? -1 : 1;
     }