Hash Base64, Sequence and cleansing, tdf#104250 prep

Change-Id: I58d48b8caa780138b8431bec9db20c9d0e9abce7

comphelper/qa/unit/test_hash.cxx

@@ -111,8 +111,8 @@ void TestHash::testSHA512_saltspin()
     const OUString aPass("pwd");
     const OUString aAlgo("SHA-512");
     const OUString aSalt("876MLoKTq42+/DLp415iZQ==");
-    const OUString aHash = comphelper::Hash::calculateHash( aPass, aSalt, 100000, aAlgo);
-    OUString aStr("5l3mgNHXpWiFaBPv5Yso1Xd/UifWvQWmlDnl/hsCYbFT2sJCzorjRmBCQ/3qeDu6Q/4+GIE8a1DsdaTwYh1q2g==");
+    const OUString aHash = comphelper::Hash::calculateHashBase64( aPass, aSalt, 100000, aAlgo);
+    const OUString aStr("5l3mgNHXpWiFaBPv5Yso1Xd/UifWvQWmlDnl/hsCYbFT2sJCzorjRmBCQ/3qeDu6Q/4+GIE8a1DsdaTwYh1q2g==");
     CPPUNIT_ASSERT_EQUAL(aStr, aHash);
 }
 

comphelper/source/misc/hash.cxx

@@ -168,7 +168,6 @@ std::vector<unsigned char> Hash::calculateHash(
         return calculateHash( pInput, nLength, eType);
 
     Hash aHash(eType);
-    std::vector<unsigned char> hash;
     if (nSaltLen)
     {
         std::vector<unsigned char> initialData( nSaltLen + nLength);
@@ -181,7 +180,7 @@ std::vector<unsigned char> Hash::calculateHash(
     {
         aHash.update( pInput, nLength);
     }
-    hash = aHash.finalize();
+    std::vector<unsigned char> hash( aHash.finalize());
 
     if (nSpinCount)
     {
@@ -232,7 +231,7 @@ std::vector<unsigned char> Hash::calculateHash(
     return calculateHash( pPassBytes, nPassBytesLen, rSaltValue.data(), rSaltValue.size(), nSpinCount, eType);
 }
 
-OUString Hash::calculateHash(
+css::uno::Sequence<sal_Int8> Hash::calculateHashSequence(
         const rtl::OUString& rPassword,
         const rtl::OUString& rSaltValue,
         sal_uInt32 nSpinCount,
@@ -248,17 +247,31 @@ OUString Hash::calculateHash(
     else if (rAlgorithmName == "MD5")
         eType = HashType::MD5;
     else
-        return OUString();
+        return css::uno::Sequence<sal_Int8>();
 
-    css::uno::Sequence<sal_Int8> aSaltSeq;
-    comphelper::Base64::decode( aSaltSeq, rSaltValue);
+    std::vector<unsigned char> aSaltVec;
+    if (!rSaltValue.isEmpty())
+    {
+        css::uno::Sequence<sal_Int8> aSaltSeq;
+        comphelper::Base64::decode( aSaltSeq, rSaltValue);
+        aSaltVec = comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq);
+    }
+
+    std::vector<unsigned char> hash( calculateHash( rPassword, aSaltVec, nSpinCount, eType));
+
+    return comphelper::containerToSequence<sal_Int8>( hash);
+}
 
-    std::vector<unsigned char> hash = calculateHash( rPassword,
-            comphelper::sequenceToContainer<std::vector<unsigned char>>( aSaltSeq),
-            nSpinCount, eType);
+OUString Hash::calculateHashBase64(
+        const rtl::OUString& rPassword,
+        const rtl::OUString& rSaltValue,
+        sal_uInt32 nSpinCount,
+        const rtl::OUString& rAlgorithmName)
+{
+    css::uno::Sequence<sal_Int8> aSeq( calculateHashSequence( rPassword, rSaltValue, nSpinCount, rAlgorithmName));
 
     OUStringBuffer aBuf;
-    comphelper::Base64::encode( aBuf, comphelper::containerToSequence<sal_Int8>( hash));
+    comphelper::Base64::encode( aBuf, aSeq);
     return aBuf.makeStringAndClear();
 }
 

include/comphelper/hash.hxx

@@ -12,6 +12,8 @@
 
 #include <comphelper/comphelperdllapi.h>
 
+#include <com/sun/star/uno/Sequence.hxx>
+
 #include <memory>
 #include <vector>
 
@@ -82,6 +84,29 @@ public:
             sal_uInt32 nSpinCount,
             HashType eType);
 
+    /** Convenience function to calculate a salted hash with iterations.
+
+        @param  rPassword
+                UTF-16LE encoded string without leading BOM character
+
+        @param  rSaltValue
+                Base64 encoded salt that will be decoded and prepended to password
+                data.
+
+        @param  rAlgorithmName
+                One of "SHA-512", "SHA-256", ... as listed in
+                https://msdn.microsoft.com/en-us/library/dd920692
+                that have a valid match in HashType. If not, an empty string is
+                returned. Not all algorithm names are supported.
+
+        @return the raw hash value as sal_Int8 sequence.
+     */
+    static css::uno::Sequence<sal_Int8> calculateHashSequence(
+            const rtl::OUString& rPassword,
+            const rtl::OUString& rSaltValue,
+            sal_uInt32 nSpinCount,
+            const rtl::OUString& rAlgorithmName);
+
     /** Convenience function to calculate a salted hash with iterations.
 
         @param  rPassword
@@ -100,7 +125,7 @@ public:
         @return the base64 encoded string of the hash value, that can be
                 compared against a stored base64 encoded hash value.
      */
-    static rtl::OUString calculateHash(
+    static rtl::OUString calculateHashBase64(
             const rtl::OUString& rPassword,
             const rtl::OUString& rSaltValue,
             sal_uInt32 nSpinCount,