add ScTokenArray::Finalize() to explicitly reduce memory usage

Since ScTokenArray::Add() overallocates memory, make sure we do not
keep such possibly large arrays. Since any copying of ScTokenArray
implicitly finalizes as well, this is not a big problem right now,
but then why needlessly do the copies? (next commit)

Change-Id: I55398bcd8fb31f1be5a4b8e3f5a71b26649a7594
Reviewed-on: https://gerrit.libreoffice.org/60862
Tested-by: Jenkins
Reviewed-by: Luboš Luňák <l.lunak@collabora.com>

formula/source/core/api/token.cxx

@@ -579,6 +579,18 @@ FormulaTokenArray::~FormulaTokenArray()
     Clear();
 }
 
+void FormulaTokenArray::Finalize()
+{
+    if( nLen && !mbFinalized )
+    {
+        // Add() overallocates, so reallocate to the minimum needed size.
+        std::unique_ptr<FormulaToken*[]> newCode(new FormulaToken*[ nLen ]);
+        std::copy(&pCode[0], &pCode[nLen], newCode.get());
+        pCode = std::move( newCode );
+        mbFinalized = true;
+    }
+}
+
 void FormulaTokenArray::Assign( const FormulaTokenArray& r )
 {
     nLen   = r.nLen;
@@ -779,7 +791,8 @@ FormulaToken* FormulaTokenArray::Add( FormulaToken* t )
 // Allocating an array of size FORMULA_MAXTOKENS is simple, but that results in relatively large
 // allocations that malloc() implementations usually do not handle as efficiently as smaller
 // sizes (not only in terms of memory usage but also speed). Since most token arrays are going
-// to be small, start with a small array and resize only if needed.
+// to be small, start with a small array and resize only if needed. Eventually Finalize() will
+// reallocate the memory to size exactly matching the requirements.
     const size_t MAX_FAST_TOKENS = 32;
     if( !pCode )
         pCode.reset(new FormulaToken*[ MAX_FAST_TOKENS ]);

include/formula/tokenarray.hxx

@@ -291,6 +291,11 @@ public:
 
     virtual void Clear();
 
+    /**
+     * The array has its final used size and no more token can be added.
+     */
+    void Finalize();
+
     void SetFromRangeName( bool b ) { mbFromRangeName = b; }
     bool IsFromRangeName() const { return mbFromRangeName; }
 

sc/source/core/data/formulacell.cxx

@@ -548,6 +548,7 @@ void ScFormulaCellGroup::setCode( ScTokenArray* pCode )
 {
     delete mpCode;
     mpCode = pCode; // takes ownership of the token array.
+    mpCode->Finalize(); // Reduce memory usage if needed.
     mbInvariant = mpCode->IsInvariant();
     mpCode->GenHash();
 }
@@ -699,6 +700,8 @@ ScFormulaCell::ScFormulaCell(
 {
     assert(pArray); // Never pass a NULL pointer here.
 
+    pCode->Finalize(); // Reduce memory usage if needed.
+
     // Generate RPN token array.
     if (pCode->GetLen() && pCode->GetCodeError() == FormulaError::NONE && !pCode->GetCodeLen())
     {
@@ -722,7 +725,7 @@ ScFormulaCell::ScFormulaCell(
     ScDocument* pDoc, const ScAddress& rPos, const ScTokenArray& rArray,
     const FormulaGrammar::Grammar eGrammar, ScMatrixMode cMatInd ) :
     eTempGrammar( eGrammar),
-    pCode(new ScTokenArray(rArray)),
+    pCode(new ScTokenArray(rArray)), // also implicitly does Finalize() on the array
     pDocument( pDoc ),
     pPrevious(nullptr),
     pNext(nullptr),