Fix vcl alpha blending n#714787

The way alpha compositing took place in vcl's alpha vdev was subtly
wrong - it was supposed to implement porter-duff 'over', but didn't.
This is now fixed also for cases where the source surface contains
alpha, and needs to blend correctly into fully-transparent
background.

For reference: http://en.wikipedia.org/wiki/Alpha_compositing

vcl/source/gdi/outdev2.cxx

@@ -1573,11 +1573,14 @@ void OutputDevice::DrawPixel( const Polygon& rPts, const Color& rColor )
 
 namespace
 {
-    sal_uInt8 lcl_calcColor( const sal_uInt8 nSourceColor, const sal_uInt8 nSourceOpaq, const sal_uInt8 nDestColor )
+    // Co = Cs + Cd*(1-As) premultiplied alpha -or-
+    // Co = (AsCs + AdCd*(1-As)) / Ao
+    inline sal_uInt8 lcl_calcColor( const sal_uInt8 nSourceColor, const sal_uInt8 nSourceAlpha,
+                                    const sal_uInt8 nDstAlpha, const sal_uInt8 nResAlpha, const sal_uInt8 nDestColor )
     {
-        int c = ( (int)nDestColor * ( 255 - nSourceOpaq ) )
-            +     (int)nSourceOpaq * (int)nSourceColor;
-        return sal_uInt8( c / 255 );
+        int c = nResAlpha ? ( (int)nSourceAlpha*nSourceColor + (int)nDstAlpha*nDestColor -
+                              (int)nDstAlpha*nDestColor*nSourceAlpha/255 ) / (int)nResAlpha : 0;
+        return sal_uInt8( c );
     }
 }
 
@@ -1631,27 +1634,29 @@ Bitmap OutputDevice::ImplBlendWithAlpha( Bitmap              aBmp,
 
                     aSrcCol = pP->GetColor( nMapY, nMapX );
                     aDstCol = pB->GetColor( nY, nX );
-                    const sal_uInt8 nSrcOpaq = 255 - pA->GetPixel( nMapY, nMapX ).GetBlueOrIndex();
-                    const sal_uInt8 nDstOpaq  = 255 - pAlphaW->GetPixel( nY, nX ).GetBlueOrIndex();
 
-                    aDstCol.SetRed( lcl_calcColor( aSrcCol.GetRed(), nSrcOpaq, aDstCol.GetRed() ) );
-                    aDstCol.SetBlue( lcl_calcColor( aSrcCol.GetBlue(), nSrcOpaq, aDstCol.GetBlue() ) );
-                    aDstCol.SetGreen( lcl_calcColor( aSrcCol.GetGreen(), nSrcOpaq, aDstCol.GetGreen() ) );
+                    // vcl stores transparency, not alpha - invert it
+                    const sal_uInt8 nSrcAlpha = 255 - pA->GetPixel( nMapY, nMapX ).GetBlueOrIndex();
+                    const sal_uInt8 nDstAlpha = 255 - pAlphaW->GetPixel( nY, nX ).GetBlueOrIndex();
+
+                    // Perform porter-duff compositing 'over' operation
+                    //
+                    // Co = Cs + Cd*(1-As)
+                    // Ad = As + Ad*(1-As)
+                    const sal_uInt8 nResAlpha = (int)nSrcAlpha + (int)nDstAlpha - (int)nDstAlpha*nSrcAlpha/255;
+
+                    aDstCol.SetRed( lcl_calcColor( aSrcCol.GetRed(), nSrcAlpha, nDstAlpha, nResAlpha, aDstCol.GetRed() ) );
+                    aDstCol.SetBlue( lcl_calcColor( aSrcCol.GetBlue(), nSrcAlpha, nDstAlpha, nResAlpha, aDstCol.GetBlue() ) );
+                    aDstCol.SetGreen( lcl_calcColor( aSrcCol.GetGreen(), nSrcAlpha, nDstAlpha, nResAlpha, aDstCol.GetGreen() ) );
 
                     aIndex.SetIndex( (sal_uInt8) ( nVCLRLut[ ( nVCLLut[ aDstCol.GetRed() ] + nD ) >> 16UL ] +
                                               nVCLGLut[ ( nVCLLut[ aDstCol.GetGreen() ] + nD ) >> 16UL ] +
                                               nVCLBLut[ ( nVCLLut[ aDstCol.GetBlue() ] + nD ) >> 16UL ] ) );
                     pW->SetPixel( nY, nX, aIndex );
 
-                    // Have to perform the compositing 'algebra' in
-                    // the inverse alpha space (with 255 meaning
-                    // opaque), otherwise, transitivity is not
-                    // achieved.
-                    const sal_uInt8 nSrcAlpha = 255-COLOR_CHANNEL_MERGE( 255, (sal_uInt8)nDstOpaq, nSrcOpaq );
-
-                    aIndex.SetIndex( (sal_uInt8) ( nVCLRLut[ ( nVCLLut[ nSrcAlpha ] + nD ) >> 16UL ] +
-                                              nVCLGLut[ ( nVCLLut[ nSrcAlpha ] + nD ) >> 16UL ] +
-                                              nVCLBLut[ ( nVCLLut[ nSrcAlpha ] + nD ) >> 16UL ] ) );
+                    aIndex.SetIndex( (sal_uInt8) ( nVCLRLut[ ( nVCLLut[ 255-nResAlpha ] + nD ) >> 16UL ] +
+                                                   nVCLGLut[ ( nVCLLut[ 255-nResAlpha ] + nD ) >> 16UL ] +
+                                                   nVCLBLut[ ( nVCLLut[ 255-nResAlpha ] + nD ) >> 16UL ] ) );
                     pAlphaW->SetPixel( nY, nX, aIndex );
                 }
             }
@@ -1676,22 +1681,23 @@ Bitmap OutputDevice::ImplBlendWithAlpha( Bitmap              aBmp,
 
                     aSrcCol = pP->GetColor( nMapY, nMapX );
                     aDstCol = pB->GetColor( nY, nX );
-                    const sal_uInt8 nSrcOpaq  = 255 - pA->GetPixel( nMapY, nMapX ).GetBlueOrIndex();
-                    const sal_uInt8 nDstOpaq  = 255 - pAlphaW->GetPixel( nY, nX ).GetBlueOrIndex();
 
-                    aDstCol.SetRed( lcl_calcColor( aSrcCol.GetRed(), nSrcOpaq, aDstCol.GetRed() ) );
-                    aDstCol.SetBlue( lcl_calcColor( aSrcCol.GetBlue(), nSrcOpaq, aDstCol.GetBlue() ) );
-                    aDstCol.SetGreen( lcl_calcColor( aSrcCol.GetGreen(), nSrcOpaq, aDstCol.GetGreen() ) );
+                    // vcl stores transparency, not alpha - invert it
+                    const sal_uInt8 nSrcAlpha = 255 - pA->GetPixel( nMapY, nMapX ).GetBlueOrIndex();
+                    const sal_uInt8 nDstAlpha = 255 - pAlphaW->GetPixel( nY, nX ).GetBlueOrIndex();
 
-                    pB->SetPixel( nY, nX, aDstCol );
+                    // Perform porter-duff compositing 'over' operation
+                    //
+                    // Co = Cs + Cd*(1-As)
+                    // Ad = As + Ad*(1-As)
+                    const sal_uInt8 nResAlpha = (int)nSrcAlpha + (int)nDstAlpha - (int)nDstAlpha*nSrcAlpha/255;
 
-                    // Have to perform the compositing 'algebra' in
-                    // the inverse alpha space (with 255 meaning
-                    // opaque), otherwise, transitivity is not
-                    // achieved.
-                    const sal_uInt8 nSrcAlpha = 255-COLOR_CHANNEL_MERGE( 255, (sal_uInt8)nDstOpaq, nSrcOpaq );
+                    aDstCol.SetRed( lcl_calcColor( aSrcCol.GetRed(), nSrcAlpha, nDstAlpha, nResAlpha, aDstCol.GetRed() ) );
+                    aDstCol.SetBlue( lcl_calcColor( aSrcCol.GetBlue(), nSrcAlpha, nDstAlpha, nResAlpha, aDstCol.GetBlue() ) );
+                    aDstCol.SetGreen( lcl_calcColor( aSrcCol.GetGreen(), nSrcAlpha, nDstAlpha, nResAlpha, aDstCol.GetGreen() ) );
 
-                    pAlphaW->SetPixel( nY, nX, Color(nSrcAlpha, nSrcAlpha, nSrcAlpha) );
+                    pB->SetPixel( nY, nX, aDstCol );
+                    pAlphaW->SetPixel( nY, nX, Color(255L-nResAlpha, 255L-nResAlpha, 255L-nResAlpha) );
                 }
             }
         }