ofz#6330 Out-of-memory

Change-Id: Ie138705e7994915d7824d8cf1ae1a0e17e247256
Reviewed-on: https://gerrit.libreoffice.org/49740Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

filter/source/graphicfilter/ipcx/ipcx.cxx

@@ -40,7 +40,7 @@ private:
     sal_uLong           nPlanes;            // no of planes
     sal_uLong           nBytesPerPlaneLin;  // bytes per plane line
 
-    sal_uLong           nWidth, nHeight;    // dimension in pixel
+    sal_uInt32          nWidth, nHeight;    // dimension in pixel
     sal_uInt16          nResX, nResY;       // resolution in pixel per inch or 0,0
     sal_uInt16          nDestBitsPerPixel;  // bits per pixel in destination bitmap 1,4,8 or 24
     std::unique_ptr<sal_uInt8[]>
@@ -96,6 +96,12 @@ bool PCXReader::ReadPCX(Graphic & rGraphic)
         bStatus = false;
     }
 
+    if (bStatus)
+    {
+        sal_uInt32 nResult;
+        bStatus = !o3tl::checked_multiply(nWidth, nHeight, nResult) && nResult <= SAL_MAX_INT32/2/3;
+    }
+
     // Write BMP header and conditionally (maybe invalid for now) color palette:
     if (bStatus)
     {