tdf#114939 sfx2: notify user of non-interoperable passwords

For ODF 1.1 encryption, SHA1 is used as the start-key digest algorithm
of PBKDF2.

For the ODF export password-to-modify "feature", PBKDF2 is used
without a start-key digest round, so the UTF-8 encoded password
is directly the input.

In both cases, if the user entered a password with a known problematic
length, reject it and request a new one.

Change-Id: Ie6510c1e668e1ed32be439ebe00354881b6ca83f

comphelper/source/misc/docpasswordhelper.cxx

@@ -49,6 +49,8 @@ static uno::Sequence< sal_Int8 > GeneratePBKDF2Hash( const OUString& aPassword,
     if ( !aPassword.isEmpty() && aSalt.getLength() && nCount && nHashLength )
     {
         OString aBytePass = OUStringToOString( aPassword, RTL_TEXTENCODING_UTF8 );
+        // FIXME this is subject to the SHA1-bug tdf#114939 - see also
+        // RequestPassword() in filedlghelper.cxx
         aResult.realloc( 16 );
         rtl_digest_PBKDF2( reinterpret_cast < sal_uInt8 * > ( aResult.getArray() ),
                            aResult.getLength(),

sfx2/source/dialog/filedlghelper.cxx

@@ -60,6 +60,7 @@
 #include <vcl/msgbox.hxx>
 #include <vcl/mnemonic.hxx>
 #include <unotools/pathoptions.hxx>
+#include <unotools/saveopt.hxx>
 #include <unotools/securityoptions.hxx>
 #include <svl/itemset.hxx>
 #include <svl/eitem.hxx>
@@ -2668,7 +2669,25 @@ ErrCode RequestPassword(const std::shared_ptr<const SfxFilter>& pCurrentFilter,
     ::rtl::Reference< ::comphelper::DocPasswordRequest > pPasswordRequest( new ::comphelper::DocPasswordRequest( eType, css::task::PasswordRequestMode_PASSWORD_CREATE, aURL, bool( pCurrentFilter->GetFilterFlags() & SfxFilterFlags::PASSWORDTOMODIFY ) ) );
 
     uno::Reference< css::task::XInteractionRequest > rRequest( pPasswordRequest.get() );
-    xInteractionHandler->handle( rRequest );
+    do {
+        xInteractionHandler->handle( rRequest );
+        if (pPasswordRequest->isPassword() && !bMSType)
+        {
+            OString const utf8Pwd(OUStringToOString(pPasswordRequest->getPassword(), RTL_TEXTENCODING_UTF8));
+            OString const utf8Ptm(OUStringToOString(pPasswordRequest->getPasswordToModify(), RTL_TEXTENCODING_UTF8));
+            if (!(52 <= utf8Pwd.getLength() && utf8Pwd.getLength() <= 55
+                    && SvtSaveOptions().GetODFDefaultVersion() < SvtSaveOptions::ODFVER_012)
+                && !(52 <= utf8Ptm.getLength() && utf8Ptm.getLength() <= 55))
+            {
+                break;
+            }
+            ScopedVclPtrInstance<MessBox>(Application::GetDefDialogParent(),
+                MessBoxStyle::Ok, 0, "Password length",
+                "The password you have entered causes interoperability issues. Please enter a password that is shorter than 52 bytes, or longer than 55 bytes."
+                )->Execute();
+
+        }
+    } while (true);
     if ( pPasswordRequest->isPassword() )
     {
         if ( pPasswordRequest->getPassword().getLength() )

uui/source/iahndl-authentication.cxx

@@ -438,6 +438,9 @@ executeMasterPasswordDialog(
     }
 
     sal_uInt8 aKey[RTL_DIGEST_LENGTH_MD5];
+    // FIXME this is subject to the SHA1-bug tdf#114939 - but this
+    // MasterPassword stuff is just stored in the UserInstallation,
+    // so no interop concerns
     rtl_digest_PBKDF2(aKey,
                       RTL_DIGEST_LENGTH_MD5,
                       reinterpret_cast< sal_uInt8 const * >(aMaster.getStr()),