Set error on more than max params (255) per function

Parameter count is size byte, so.. SUM(1,1,1,...) with 256 arguments resulted
in 0 (uint8 wrapping around).

(cherry picked from commit 209cc5c2)

Change-Id: Ib9997ad0d0d13d4c5171f276148b6c5cad570d5b
Reviewed-on: https://gerrit.libreoffice.org/39506Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

formula/source/core/api/FormulaCompiler.cxx

@@ -1511,7 +1511,7 @@ void FormulaCompiler::Factor()
                 }
                 else
                     SetError( FormulaError::PairExpected);
-                sal_uInt8 nSepCount = 0;
+                sal_uInt32 nSepCount = 0;
                 const sal_uInt16 nSepPos = pArr->nIndex - 1;    // separator position, if any
                 if( !bNoParam )
                 {
@@ -1521,6 +1521,8 @@ void FormulaCompiler::Factor()
                         NextToken();
                         CheckSetForceArrayParameter( mpToken, nSepCount);
                         nSepCount++;
+                        if (nSepCount > FORMULA_MAXPARAMS)
+                            SetError( FormulaError::CodeOverflow);
                         eOp = Expression();
                     }
                 }
@@ -1617,7 +1619,7 @@ void FormulaCompiler::Factor()
             }
             else
                 SetError( FormulaError::PairExpected);
-            sal_uInt8 nSepCount = 0;
+            sal_uInt32 nSepCount = 0;
             if( !bNoParam )
             {
                 nSepCount++;
@@ -1626,6 +1628,8 @@ void FormulaCompiler::Factor()
                     NextToken();
                     CheckSetForceArrayParameter( mpToken, nSepCount);
                     nSepCount++;
+                    if (nSepCount > FORMULA_MAXPARAMS)
+                        SetError( FormulaError::CodeOverflow);
                     eOp = Expression();
                 }
             }

include/formula/FormulaCompiler.hxx

@@ -38,6 +38,7 @@
 
 #define FORMULA_MAXJUMPCOUNT    32  /* maximum number of jumps (ocChoose) */
 #define FORMULA_MAXTOKENS     8192  /* maximum number of tokens in formula */
+#define FORMULA_MAXPARAMS      255  /* maximum number of parameters per function (byte) */
 
 
 namespace com { namespace sun { namespace star {