ofz#3540: oom move compression size test earlier

Change-Id: I1ca4f08ef526498cf08665f2b23d56095d5f1323
Reviewed-on: https://gerrit.libreoffice.org/43082Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

filter/source/graphicfilter/itiff/itiff.cxx

@@ -1433,6 +1433,13 @@ bool TIFFReader::ReadTIFF(SvStream & rTIFF, Graphic & rGraphic )
                         if (nStart > nEndOfFile)
                             bStatus = false;
                     }
+
+                    if (bStatus)
+                    {
+                        sal_uLong nTargetBits = nImageWidth * nBitsPerSample * nSamplesPerPixel / nPlanes;
+                        if (nTargetBits > SAL_MAX_UINT16)
+                            bStatus = false;
+                    }
                 }
                 else if ( nCompression == 5 )
                 {