ofz#6334 Out-of-memory

Change-Id: I714d4b3c767a2e9244e4d5a0b608e3d7fd3002a3
Reviewed-on: https://gerrit.libreoffice.org/49741Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

filter/source/graphicfilter/ipsd/ipsd.cxx

@@ -107,6 +107,13 @@ bool PSDReader::ReadPSD(Graphic & rGraphic )
     if ( !ImplReadHeader() )
         return false;
 
+    if (mbStatus)
+    {
+        sal_uInt32 nResult;
+        if (o3tl::checked_multiply(mpFileHeader->nColumns, mpFileHeader->nRows, nResult) || nResult > SAL_MAX_INT32/2/3)
+            return false;
+    }
+
     Size aBitmapSize( mpFileHeader->nColumns, mpFileHeader->nRows );
     mpBitmap.reset( new vcl::bitmap::RawBitmap( aBitmapSize ) );
     if ( mpPalette && mbStatus )