AddressSanitizer: undefined-behavior

Change-Id: I55a92512ad9e1508c49ee3149394196f5be5f162
Reviewed-on: https://gerrit.libreoffice.org/42784Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

sot/source/sdstor/stgio.cxx

@@ -24,6 +24,7 @@
 #include "stgstrms.hxx"
 #include "stgdir.hxx"
 #include "stgio.hxx"
+#include <o3tl/safeint.hxx>
 #include <rtl/instance.hxx>
 
 #include <memory>
@@ -88,9 +89,21 @@ void StgIo::SetupStreams()
     m_pDataStrm = nullptr;
     m_pFAT      = nullptr;
     ResetError();
-    SetPhysPageSize( 1 << m_aHdr.GetPageSize() );
-    m_pFAT = new StgFATStrm( *this );
-    m_pTOC = new StgDirStrm( *this );
+    short nPhysPageSize = 1 << m_aHdr.GetPageSize();
+    SetPhysPageSize(nPhysPageSize);
+    sal_Int32 nFatStrmSize;
+    if (o3tl::checked_multiply<sal_Int32>(m_aHdr.GetFATSize(), nPhysPageSize, nFatStrmSize))
+    {
+        SAL_WARN("sot", "Error: " << m_aHdr.GetFATSize() << " * " << nPhysPageSize << " would overflow");
+        SetError(SVSTREAM_FILEFORMAT_ERROR);
+        m_pFAT = nullptr;
+        m_pTOC = nullptr;
+    }
+    else
+    {
+        m_pFAT = new StgFATStrm(*this, nFatStrmSize);
+        m_pTOC = new StgDirStrm( *this );
+    }
     if( !GetError() )
     {
         StgDirEntry* pRoot = m_pTOC->GetRoot();

sot/source/sdstor/stgstrms.cxx

@@ -572,10 +572,10 @@ bool StgStrm::SetSize( sal_Int32 nBytes )
 // Since this access is implemented as a StgStrm, we can use the
 // FAT allocator.
 
-StgFATStrm::StgFATStrm( StgIo& r ) : StgStrm( r )
+StgFATStrm::StgFATStrm(StgIo& r, sal_Int32 nFatStrmSize) : StgStrm( r )
 {
     m_pFat.reset( new StgFAT( *this, true ) );
-    m_nSize = m_rIo.m_aHdr.GetFATSize() * m_nPageSize;
+    m_nSize = nFatStrmSize;
 }
 
 bool StgFATStrm::Pos2Page( sal_Int32 nBytePos )

sot/source/sdstor/stgstrms.hxx

@@ -101,7 +101,7 @@ class StgFATStrm : public StgStrm {     // the master FAT stream
     virtual bool Pos2Page( sal_Int32 nBytePos ) override;
     bool  SetPage( short, sal_Int32 );
 public:
-    explicit StgFATStrm( StgIo& );
+    explicit StgFATStrm(StgIo&, sal_Int32 nFatStrmSize);
     using StgStrm::GetPage;
     sal_Int32 GetPage( short, bool, sal_uInt16 *pnMasterAlloc = nullptr);
     virtual bool SetSize( sal_Int32 ) override;