implement CryptoAPI RC4+SHA1 encryption scheme for xls import

there might be other variants out there in practice, but this
works for default encrypted xls of excel 2013

Change-Id: I91c0e1d1d95fbd1c68966650e7ac7d23276bcbe3

include/filter/msfilter/mscodec.hxx

@@ -25,6 +25,7 @@
 #include <rtl/cipher.h>
 #include <rtl/digest.h>
 #include <sal/types.h>
+#include <vector>
 
 namespace com { namespace sun { namespace star {
     namespace beans { struct NamedValue; }
@@ -177,7 +178,7 @@ public:
 class MSFILTER_DLLPUBLIC MSCodec97
 {
 public:
-    MSCodec97(rtlCipher m_hCipher);
+    MSCodec97(size_t nHashLen);
     virtual ~MSCodec97();
 
     /** Initializes the algorithm with the encryption data.
@@ -186,7 +187,7 @@ public:
             The sequence contains the necessary data to initialize
             the codec.
      */
-    virtual bool InitCodec(const css::uno::Sequence< css::beans::NamedValue >& aData) = 0;
+    bool InitCodec(const css::uno::Sequence< css::beans::NamedValue >& aData);
 
     /** Retrieves the encryption data
 
@@ -194,8 +195,37 @@ public:
             The sequence contains the necessary data to initialize
             the codec.
      */
-    virtual css::uno::Sequence< css::beans::NamedValue > GetEncryptionData() = 0;
+    css::uno::Sequence< css::beans::NamedValue > GetEncryptionData();
+
+    /** Initializes the algorithm with the specified password and document ID.
+
+        @param pPassData
+            Wide character array containing the password. Must be zero
+            terminated, which results in a maximum length of 15 characters.
+        @param pDocId
+            Unique document identifier read from or written to the file.
+     */
+    virtual void InitKey(const sal_uInt16 pPassData[16],
+                         const sal_uInt8 pDocId[16]) = 0;
+
+
+    /** Verifies the validity of the password using the passed salt data.
+
+        @precond
+            The codec must be initialized with InitKey() before this function
+            can be used.
+
+        @param pSaltData
+            Salt data block read from the file.
+        @param pSaltDigest
+            Salt digest read from the file.
+
+        @return
+            true = Test was successful.
+     */
+    bool VerifyKey(const sal_uInt8* pSaltData, const sal_uInt8* pSaltDigest);
 
+    virtual void GetDigestFromSalt(const sal_uInt8* pSaltData, sal_uInt8* pDigest) = 0;
 
     /** Rekeys the codec using the specified counter.
 
@@ -278,12 +308,19 @@ public:
      */
     bool                Skip(std::size_t nDatLen);
 
+    /* allows to get the unique document id from the codec
+     */
+    void                GetDocId( sal_uInt8 pDocId[16] );
+
 private:
                         MSCodec97(const MSCodec97&) = delete;
     MSCodec97&          operator=(const MSCodec97&) = delete;
 
 protected:
+    size_t              m_nHashLen;
     rtlCipher           m_hCipher;
+    sal_uInt8           m_pDocId[16];
+    std::vector<sal_uInt8> m_aDigestValue;
 };
 
 /** Encodes and decodes data from protected MSO 97+ documents.
@@ -295,25 +332,8 @@ protected:
 class MSFILTER_DLLPUBLIC MSCodec_Std97 :  public MSCodec97
 {
 public:
-    explicit            MSCodec_Std97();
-                        ~MSCodec_Std97();
-
-    /** Initializes the algorithm with the encryption data.
-
-        @param aData
-            The sequence contains the necessary data to initialize
-            the codec.
-     */
-    virtual bool InitCodec(const css::uno::Sequence< css::beans::NamedValue >& aData) override;
-
-    /** Retrieves the encryption data
-
-        @return
-            The sequence contains the necessary data to initialize
-            the codec.
-     */
-    virtual css::uno::Sequence<css::beans::NamedValue> GetEncryptionData() override;
-
+    MSCodec_Std97();
+    virtual ~MSCodec_Std97() override;
 
     /** Initializes the algorithm with the specified password and document ID.
 
@@ -323,27 +343,8 @@ public:
         @param pDocId
             Unique document identifier read from or written to the file.
      */
-    void                InitKey(
-                            const sal_uInt16 pPassData[ 16 ],
-                            const sal_uInt8 pDocId[ 16 ] );
-
-    /** Verifies the validity of the password using the passed salt data.
-
-        @precond
-            The codec must be initialized with InitKey() before this function
-            can be used.
-
-        @param pSaltData
-            Salt data block read from the file.
-        @param pSaltDigest
-            Salt digest read from the file.
-
-        @return
-            true = Test was successful.
-     */
-    bool                VerifyKey(
-                            const sal_uInt8 pSaltData[ 16 ],
-                            const sal_uInt8 pSaltDigest[ 16 ] );
+    virtual void InitKey(const sal_uInt16 pPassData[16],
+                         const sal_uInt8 pDocId[16]) override;
 
     /** Rekeys the codec using the specified counter.
 
@@ -384,19 +385,24 @@ public:
                             sal_uInt8 pSaltData[16],
                             sal_uInt8 pSaltDigest[16]);
 
-    /* allows to get the unique document id from the codec
-     */
-    void                GetDocId( sal_uInt8 pDocId[16] );
-
-    void                GetDigestFromSalt( const sal_uInt8 pSaltData[16], sal_uInt8 pDigest[16] );
+    virtual void        GetDigestFromSalt(const sal_uInt8* pSaltData, sal_uInt8* pDigest) override;
 
 private:
                         MSCodec_Std97( const MSCodec_Std97& ) = delete;
     MSCodec_Std97&      operator=( const MSCodec_Std97& ) = delete;
 
     rtlDigest           m_hDigest;
-    sal_uInt8           m_pDigestValue[ RTL_DIGEST_LENGTH_MD5 ];
-    sal_uInt8           m_pDocId[16];
+};
+
+class MSFILTER_DLLPUBLIC MSCodec_CryptoAPI :  public MSCodec97
+{
+public:
+    MSCodec_CryptoAPI();
+
+    virtual void InitKey(const sal_uInt16 pPassData[16],
+                         const sal_uInt8 pDocId[16]) override;
+    virtual bool InitCipher(sal_uInt32 nCounter) override;
+    virtual void GetDigestFromSalt(const sal_uInt8* pSaltData, sal_uInt8* pDigest) override;
 };
 
 const sal_uInt32 ENCRYPTINFO_CRYPTOAPI      = 0x00000004;
@@ -459,12 +465,28 @@ struct MSFILTER_DLLPUBLIC EncryptionVerifierAES
     EncryptionVerifierAES();
 };
 
+struct MSFILTER_DLLPUBLIC EncryptionVerifierRC4
+{
+    sal_uInt32 saltSize;                                                // must be 0x00000010
+    sal_uInt8  salt[SALT_LENGTH];                                       // random generated salt value
+    sal_uInt8  encryptedVerifier[ENCRYPTED_VERIFIER_LENGTH];            // randomly generated verifier value
+    sal_uInt32 encryptedVerifierHashSize;                               // actually written hash size - depends on algorithm
+    sal_uInt8  encryptedVerifierHash[SHA1_HASH_LENGTH];                 // verifier value hash - itself also encrypted
+
+    EncryptionVerifierRC4();
+};
+
 struct MSFILTER_DLLPUBLIC StandardEncryptionInfo
 {
     EncryptionStandardHeader header;
     EncryptionVerifierAES    verifier;
 };
 
+struct MSFILTER_DLLPUBLIC RC4EncryptionInfo
+{
+    EncryptionStandardHeader header;
+    EncryptionVerifierRC4 verifier;
+};
 
 } // namespace msfilter
 

sc/source/filter/excel/xicontent.cxx

@@ -64,6 +64,7 @@
 #include <memory>
 #include <utility>
 #include <o3tl/make_unique.hxx>
+#include <oox/helper/helper.hxx>
 
 using ::com::sun::star::uno::Sequence;
 using ::std::unique_ptr;
@@ -1110,21 +1111,80 @@ XclImpDecrypterRef lclReadFilepass8_Standard( XclImpStream& rStrm )
     OSL_ENSURE( rStrm.GetRecLeft() == 48, "lclReadFilepass8 - wrong record size" );
     if( rStrm.GetRecLeft() == 48 )
     {
-        sal_uInt8 pnSalt[ 16 ];
-        sal_uInt8 pnVerifier[ 16 ];
-        sal_uInt8 pnVerifierHash[ 16 ];
-        rStrm.Read( pnSalt, 16 );
-        rStrm.Read( pnVerifier, 16 );
-        rStrm.Read( pnVerifierHash, 16 );
-        xDecr.reset( new XclImpBiff8Decrypter( pnSalt, pnVerifier, pnVerifierHash ) );
+        std::vector<sal_uInt8> aSalt(16);
+        std::vector<sal_uInt8> aVerifier(16);
+        std::vector<sal_uInt8> aVerifierHash(16);
+        rStrm.Read(aSalt.data(), 16);
+        rStrm.Read(aVerifier.data(), 16);
+        rStrm.Read(aVerifierHash.data(), 16);
+        xDecr.reset(new XclImpBiff8StdDecrypter(aSalt, aVerifier, aVerifierHash));
     }
     return xDecr;
 }
 
-XclImpDecrypterRef lclReadFilepass8_Strong( XclImpStream& /*rStrm*/ )
+XclImpDecrypterRef lclReadFilepass8_Strong(XclImpStream& rStream)
 {
-    // not supported
-    return XclImpDecrypterRef();
+    //Its possible there are other variants in existance but these
+    //are the defaults I get with Excel 2013
+    XclImpDecrypterRef xDecr;
+
+    msfilter::RC4EncryptionInfo info;
+
+    info.header.flags = rStream.ReaduInt32();
+    if (oox::getFlag( info.header.flags, msfilter::ENCRYPTINFO_EXTERNAL))
+        return xDecr;
+
+    sal_uInt32 nHeaderSize = rStream.ReaduInt32();
+    sal_uInt32 actualHeaderSize = sizeof(info.header);
+
+    if( (nHeaderSize < actualHeaderSize) )
+        return xDecr;
+
+    info.header.flags = rStream.ReaduInt32();
+    info.header.sizeExtra = rStream.ReaduInt32();
+    info.header.algId = rStream.ReaduInt32();
+    info.header.algIdHash = rStream.ReaduInt32();
+    info.header.keyBits = rStream.ReaduInt32();
+    info.header.providedType = rStream.ReaduInt32();
+    info.header.reserved1 = rStream.ReaduInt32();
+    info.header.reserved2 = rStream.ReaduInt32();
+
+    rStream.Ignore(nHeaderSize - actualHeaderSize);
+
+    info.verifier.saltSize = rStream.ReaduInt32();
+    if (info.verifier.saltSize != 16)
+        return xDecr;
+    rStream.Read(&info.verifier.salt, sizeof(info.verifier.salt));
+    rStream.Read(&info.verifier.encryptedVerifier, sizeof(info.verifier.encryptedVerifier));
+
+    info.verifier.encryptedVerifierHashSize = rStream.ReaduInt32();
+    if (info.verifier.encryptedVerifierHashSize != RTL_DIGEST_LENGTH_SHA1)
+        return xDecr;
+    rStream.Read(&info.verifier.encryptedVerifierHash, info.verifier.encryptedVerifierHashSize);
+
+    // check flags and algorithm IDs, required are AES128 and SHA-1
+    if (!oox::getFlag(info.header.flags, msfilter::ENCRYPTINFO_CRYPTOAPI))
+        return xDecr;
+
+    if (oox::getFlag(info.header.flags, msfilter::ENCRYPTINFO_AES))
+        return xDecr;
+
+    if (info.header.algId != msfilter::ENCRYPT_ALGO_RC4)
+        return xDecr;
+
+    // hash algorithm ID 0 defaults to SHA-1 too
+    if (info.header.algIdHash != 0 && info.header.algIdHash != msfilter::ENCRYPT_HASH_SHA1)
+        return xDecr;
+
+    xDecr.reset(new XclImpBiff8CryptoAPIDecrypter(
+        std::vector<sal_uInt8>(info.verifier.salt,
+            info.verifier.salt + SAL_N_ELEMENTS(info.verifier.salt)),
+        std::vector<sal_uInt8>(info.verifier.encryptedVerifier,
+            info.verifier.encryptedVerifier + SAL_N_ELEMENTS(info.verifier.encryptedVerifier)),
+        std::vector<sal_uInt8>(info.verifier.encryptedVerifierHash,
+            info.verifier.encryptedVerifierHash + SAL_N_ELEMENTS(info.verifier.encryptedVerifierHash))));
+
+    return xDecr;
 }
 
 XclImpDecrypterRef lclReadFilepass8( XclImpStream& rStrm )

sc/source/filter/excel/xistream.cxx

@@ -192,28 +192,50 @@ sal_uInt16 XclImpBiff5Decrypter::OnRead( SvStream& rStrm, sal_uInt8* pnData, sal
     return nRet;
 }
 
-XclImpBiff8Decrypter::XclImpBiff8Decrypter( sal_uInt8 pnSalt[ 16 ],
-        sal_uInt8 pnVerifier[ 16 ], sal_uInt8 pnVerifierHash[ 16 ] ) :
-    maSalt( pnSalt, pnSalt + 16 ),
-    maVerifier( pnVerifier, pnVerifier + 16 ),
-    maVerifierHash( pnVerifierHash, pnVerifierHash + 16 )
+XclImpBiff8Decrypter::XclImpBiff8Decrypter(const std::vector<sal_uInt8>& rSalt,
+                                           const std::vector<sal_uInt8>& rVerifier,
+                                           const std::vector<sal_uInt8>& rVerifierHash)
+    : maSalt(rSalt)
+    , maVerifier(rVerifier)
+    , maVerifierHash(rVerifierHash)
+    , mpCodec(nullptr)
 {
 }
 
-XclImpBiff8Decrypter::XclImpBiff8Decrypter( const XclImpBiff8Decrypter& rSrc ) :
-    XclImpDecrypter( rSrc ),
-    maEncryptionData( rSrc.maEncryptionData ),
-    maSalt( rSrc.maSalt ),
-    maVerifier( rSrc.maVerifier ),
-    maVerifierHash( rSrc.maVerifierHash )
+XclImpBiff8Decrypter::XclImpBiff8Decrypter(const XclImpBiff8Decrypter& rSrc)
+    : XclImpDecrypter(rSrc)
+    , maEncryptionData(rSrc.maEncryptionData)
+    , maSalt(rSrc.maSalt)
+    , maVerifier(rSrc.maVerifier)
+    , maVerifierHash(rSrc.maVerifierHash)
+    , mpCodec(nullptr)
 {
-    if( IsValid() )
-        maCodec.InitCodec( maEncryptionData );
 }
 
-XclImpBiff8Decrypter* XclImpBiff8Decrypter::OnClone() const
+XclImpBiff8StdDecrypter::XclImpBiff8StdDecrypter(const XclImpBiff8StdDecrypter& rSrc)
+    : XclImpBiff8Decrypter(rSrc)
 {
-    return new XclImpBiff8Decrypter( *this );
+    mpCodec = &maCodec;
+    if (IsValid())
+        maCodec.InitCodec(maEncryptionData);
+}
+
+XclImpBiff8StdDecrypter* XclImpBiff8StdDecrypter::OnClone() const
+{
+    return new XclImpBiff8StdDecrypter(*this);
+}
+
+XclImpBiff8CryptoAPIDecrypter::XclImpBiff8CryptoAPIDecrypter(const XclImpBiff8CryptoAPIDecrypter& rSrc)
+    : XclImpBiff8Decrypter(rSrc)
+{
+    mpCodec = &maCodec;
+    if (IsValid())
+        maCodec.InitCodec(maEncryptionData);
+}
+
+XclImpBiff8CryptoAPIDecrypter* XclImpBiff8CryptoAPIDecrypter::OnClone() const
+{
+    return new XclImpBiff8CryptoAPIDecrypter(*this);
 }
 
 uno::Sequence< beans::NamedValue > XclImpBiff8Decrypter::OnVerifyPassword( const OUString& rPassword )
@@ -232,9 +254,9 @@ uno::Sequence< beans::NamedValue > XclImpBiff8Decrypter::OnVerifyPassword( const
             *aIt = static_cast< sal_uInt16 >( *pcChar );
 
         // init codec
-        maCodec.InitKey( &aPassVect.front(), &maSalt.front() );
-        if ( maCodec.VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
-            maEncryptionData = maCodec.GetEncryptionData();
+        mpCodec->InitKey( &aPassVect.front(), &maSalt.front() );
+        if ( mpCodec->VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
+            maEncryptionData = mpCodec->GetEncryptionData();
     }
 
     return maEncryptionData;
@@ -247,9 +269,9 @@ bool XclImpBiff8Decrypter::OnVerifyEncryptionData( const uno::Sequence< beans::N
     if( rEncryptionData.getLength() )
     {
         // init codec
-        maCodec.InitCodec( rEncryptionData );
+        mpCodec->InitCodec( rEncryptionData );
 
-        if ( maCodec.VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
+        if ( mpCodec->VerifyKey( &maVerifier.front(), &maVerifierHash.front() ) )
             maEncryptionData = rEncryptionData;
     }
 
@@ -269,13 +291,13 @@ void XclImpBiff8Decrypter::OnUpdate( std::size_t nOldStrmPos, std::size_t nNewSt
         /*  Rekey cipher, if block changed or if previous offset in same block. */
         if( (nNewBlock != nOldBlock) || (nNewOffset < nOldOffset) )
         {
-            maCodec.InitCipher( nNewBlock );
+            mpCodec->InitCipher( nNewBlock );
             nOldOffset = 0;     // reset nOldOffset for next if() statement
         }
 
         /*  Seek to correct offset. */
         if( nNewOffset > nOldOffset )
-            maCodec.Skip( nNewOffset - nOldOffset );
+            mpCodec->Skip( nNewOffset - nOldOffset );
     }
 }
 
@@ -293,9 +315,9 @@ sal_uInt16 XclImpBiff8Decrypter::OnRead( SvStream& rStrm, sal_uInt8* pnData, sal
         // read the block from stream
         nRet = nRet + static_cast<sal_uInt16>(rStrm.ReadBytes(pnCurrData, nDecBytes));
         // decode the block inplace
-        maCodec.Decode( pnCurrData, nDecBytes, pnCurrData, nDecBytes );
+        mpCodec->Decode( pnCurrData, nDecBytes, pnCurrData, nDecBytes );
         if( GetOffset( rStrm.Tell() ) == 0 )
-            maCodec.InitCipher( GetBlock( rStrm.Tell() ) );
+            mpCodec->InitCipher( GetBlock( rStrm.Tell() ) );
 
         pnCurrData += nDecBytes;
         nBytesLeft = nBytesLeft - nDecBytes;

sc/source/filter/inc/xistream.hxx

@@ -119,16 +119,7 @@ private:
 /** Decrypts BIFF8 stream contents using the given document identifier. */
 class XclImpBiff8Decrypter : public XclImpDecrypter
 {
-public:
-    explicit            XclImpBiff8Decrypter( sal_uInt8 pnSalt[ 16 ],
-                            sal_uInt8 pnVerifier[ 16 ], sal_uInt8 pnVerifierHash[ 16 ] );
-
 private:
-    /** Private copy c'tor for OnClone(). */
-    explicit            XclImpBiff8Decrypter( const XclImpBiff8Decrypter& rSrc );
-
-    /** Implementation of cloning this object. */
-    virtual XclImpBiff8Decrypter* OnClone() const override;
     /** Implements password verification and initialization of the decoder. */
     virtual css::uno::Sequence< css::beans::NamedValue >
         OnVerifyPassword( const OUString& rPassword ) override;
@@ -143,12 +134,62 @@ private:
     /** Returns the block offset corresponding to the passed stream position. */
     static sal_uInt16   GetOffset( std::size_t nStrmPos );
 
+protected:
+    explicit  XclImpBiff8Decrypter(const std::vector<sal_uInt8>& rSalt,
+                                   const std::vector<sal_uInt8>& rVerifier,
+                                   const std::vector<sal_uInt8>& rVerifierHash);
+
+    explicit  XclImpBiff8Decrypter(const XclImpBiff8Decrypter& rSrc);
+
+    css::uno::Sequence< css::beans::NamedValue > maEncryptionData;
+    std::vector< sal_uInt8 > maSalt;
+    std::vector< sal_uInt8 > maVerifier;
+    std::vector< sal_uInt8 > maVerifierHash;
+    msfilter::MSCodec97* mpCodec;       /// Crypto algorithm implementation.
+};
+
+class XclImpBiff8StdDecrypter : public XclImpBiff8Decrypter
+{
+public:
+    explicit XclImpBiff8StdDecrypter(const std::vector<sal_uInt8>& rSalt,
+                                     const std::vector<sal_uInt8>& rVerifier,
+                                     const std::vector<sal_uInt8>& rVerifierHash)
+        : XclImpBiff8Decrypter(rSalt, rVerifier, rVerifierHash)
+    {
+        mpCodec = &maCodec;
+    }
+
+private:
+    /** Private copy c'tor for OnClone(). */
+    explicit XclImpBiff8StdDecrypter(const XclImpBiff8StdDecrypter& rSrc);
+
+    /** Implementation of cloning this object. */
+    virtual XclImpBiff8StdDecrypter* OnClone() const override;
+
 private:
     ::msfilter::MSCodec_Std97 maCodec;       /// Crypto algorithm implementation.
-    css::uno::Sequence< css::beans::NamedValue > maEncryptionData;
-    ::std::vector< sal_uInt8 > maSalt;
-    ::std::vector< sal_uInt8 > maVerifier;
-    ::std::vector< sal_uInt8 > maVerifierHash;
+};
+
+class XclImpBiff8CryptoAPIDecrypter : public XclImpBiff8Decrypter
+{
+public:
+    explicit XclImpBiff8CryptoAPIDecrypter(const std::vector<sal_uInt8>& rSalt,
+                                           const std::vector<sal_uInt8>& rVerifier,
+                                           const std::vector<sal_uInt8>& rVerifierHash)
+        : XclImpBiff8Decrypter(rSalt, rVerifier, rVerifierHash)
+    {
+        mpCodec = &maCodec;
+    }
+
+private:
+    /** Private copy c'tor for OnClone(). */
+    explicit XclImpBiff8CryptoAPIDecrypter(const XclImpBiff8CryptoAPIDecrypter& rSrc);
+
+    /** Implementation of cloning this object. */
+    virtual XclImpBiff8CryptoAPIDecrypter* OnClone() const override;
+
+private:
+    ::msfilter::MSCodec_CryptoAPI maCodec;       /// Crypto algorithm implementation.
 };
 
 // Stream