Deduplicate conditional formats loaded from .ods

If there are several separate conditional format elements that can be
represented as just one (with several ranges), try to do that.

A particular customer document used to take 3 minutes 20 seconds to
load, and it contained so many (tens of thousands) conditional formats
that the Format> Conditional Formatting> Manage... dialog was
practically impossible to use.

Now loading that document takes 15 seconds and there are just a
handful of separate conditional formats.

Also add a simple unit test to verify the deduplication.

Change-Id: I7c468af99956d4646ee5507390f1476caff52325
Reviewed-on: https://gerrit.libreoffice.org/45460Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Tor Lillqvist <tml@collabora.com>

sc/CppunitTest_sc_cond_format_merge.mk

+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+$(eval $(call gb_CppunitTest_CppunitTest,sc_cond_format_merge))
+
+$(eval $(call gb_CppunitTest_add_exception_objects,sc_cond_format_merge, \
+    sc/qa/unit/cond_format_merge \
+))
+
+$(eval $(call gb_CppunitTest_use_externals,sc_cond_format_merge, \
+    boost_headers \
+))
+
+$(eval $(call gb_CppunitTest_use_libraries,sc_cond_format_merge, \
+    basegfx \
+    comphelper \
+    cppu \
+    cppuhelper \
+    drawinglayer \
+    editeng \
+    for \
+    forui \
+    i18nlangtag \
+    msfilter \
+    oox \
+    sal \
+    salhelper \
+    sax \
+    sb \
+    sc \
+    scqahelper \
+    sfx \
+    sot \
+    subsequenttest \
+    svl \
+    svt \
+    svx \
+    svxcore \
+    test \
+    tk \
+    tl \
+    ucbhelper \
+    unotest \
+    utl \
+    vbahelper \
+    vcl \
+    xo \
+))
+
+$(eval $(call gb_CppunitTest_set_include,sc_cond_format_merge,\
+    -I$(SRCDIR)/sc/source/ui/inc \
+    -I$(SRCDIR)/sc/inc \
+    $$(INCLUDE) \
+))
+
+$(eval $(call gb_CppunitTest_use_sdk_api,sc_cond_format_merge))
+
+$(eval $(call gb_CppunitTest_use_ure,sc_cond_format_merge))
+$(eval $(call gb_CppunitTest_use_vcl,sc_cond_format_merge))
+
+$(eval $(call gb_CppunitTest_use_components,sc_cond_format_merge,\
+    basic/util/sb \
+    chart2/source/chartcore \
+    chart2/source/controller/chartcontroller \
+    comphelper/util/comphelp \
+    configmgr/source/configmgr \
+    dbaccess/util/dba \
+    embeddedobj/util/embobj \
+    eventattacher/source/evtatt \
+    filter/source/config/cache/filterconfig1 \
+    filter/source/storagefilterdetect/storagefd \
+    forms/util/frm \
+    framework/util/fwk \
+    i18npool/util/i18npool \
+    oox/util/oox \
+    package/source/xstor/xstor \
+    package/util/package2 \
+    sax/source/expatwrap/expwrap \
+    scaddins/source/analysis/analysis \
+    scaddins/source/datefunc/date \
+    scripting/source/basprov/basprov \
+    scripting/util/scriptframe \
+    sc/util/sc \
+    sc/util/scd \
+    sc/util/scfilt \
+    $(call gb_Helper_optional,SCRIPTING, \
+        sc/util/vbaobj) \
+    sfx2/util/sfx \
+    sot/util/sot \
+    svl/source/fsstor/fsstorage \
+    svl/util/svl \
+    svtools/util/svt \
+    svx/util/svx \
+    svx/util/svxcore \
+    toolkit/util/tk \
+    ucb/source/core/ucb1 \
+    ucb/source/ucp/file/ucpfile1 \
+    ucb/source/ucp/tdoc/ucptdoc1 \
+    unotools/util/utl \
+    unoxml/source/rdf/unordf \
+    unoxml/source/service/unoxml \
+	uui/util/uui \
+    xmloff/util/xo \
+))
+
+$(eval $(call gb_CppunitTest_use_configuration,sc_cond_format_merge))
+
+$(eval $(call gb_CppunitTest_use_unittest_configuration,sc_cond_format_merge))
+
+# vim: set noet sw=4 ts=4:

sc/Module_sc.mk

@@ -57,6 +57,7 @@ endif
 endif
 
 $(eval $(call gb_Module_add_slowcheck_targets,sc, \
+	CppunitTest_sc_cond_format_merge \
 	CppunitTest_sc_new_cond_format_api \
 	CppunitTest_sc_subsequent_filters_test \
 	CppunitTest_sc_subsequent_export_test \

sc/inc/conditio.hxx

@@ -352,6 +352,8 @@ public:
 
     bool            operator== ( const ScConditionEntry& r ) const;
 
+    bool            EqualIgnoringSrcPos( const ScConditionEntry& r ) const;
+
     virtual void SetParent( ScConditionalFormat* pNew ) override;
 
     bool IsCellValid( ScRefCellValue& rCell, const ScAddress& rPos ) const;
@@ -360,6 +362,7 @@ public:
     void SetOperation(ScConditionMode eMode);
     bool            IsIgnoreBlank() const       { return ( nOptions & SC_COND_NOBLANKS ) == 0; }
     void            SetIgnoreBlank(bool bSet);
+    OUString        GetSrcString() const         { return aSrcString; }
     const ScAddress& GetSrcPos() const           { return aSrcPos; }
 
     ScAddress       GetValidSrcPos() const;     // adjusted to allow textual representation of expressions

sc/qa/unit/cond_format_merge.cxx

+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <sal/config.h>
+
+#include <com/sun/star/beans/XPropertySet.hpp>
+#include <com/sun/star/lang/XComponent.hpp>
+#include <com/sun/star/sheet/XConditionalFormats.hpp>
+#include <com/sun/star/sheet/XSpreadsheet.hpp>
+#include <test/bootstrapfixture.hxx>
+#include <test/calc_unoapi_test.hxx>
+
+#include <global.hxx>
+#include <document.hxx>
+
+#include "helper/qahelper.hxx"
+
+using namespace css;
+
+class ScCondFormatMergeTest : public CalcUnoApiTest
+{
+public:
+    ScCondFormatMergeTest();
+
+    void testCondFormatMerge();
+
+    CPPUNIT_TEST_SUITE(ScCondFormatMergeTest);
+    CPPUNIT_TEST(testCondFormatMerge);
+    CPPUNIT_TEST_SUITE_END();
+};
+
+ScCondFormatMergeTest::ScCondFormatMergeTest()
+    : CalcUnoApiTest("sc/qa/extras/testdocuments/")
+{
+}
+
+void ScCondFormatMergeTest::testCondFormatMerge()
+{
+    OUString aFileURL;
+    createFileURL("cond_format_merge.ods", aFileURL);
+    uno::Reference<lang::XComponent> mxComponent = loadFromDesktop(aFileURL);
+
+    CPPUNIT_ASSERT_MESSAGE("Component not loaded", mxComponent.is());
+
+    // get the first sheet
+    uno::Reference<sheet::XSpreadsheetDocument> xDoc(mxComponent, uno::UNO_QUERY_THROW);
+    uno::Reference<container::XIndexAccess> xIndex(xDoc->getSheets(), uno::UNO_QUERY_THROW);
+    uno::Reference<sheet::XSpreadsheet> xSheet(xIndex->getByIndex(0), uno::UNO_QUERY_THROW);
+
+    uno::Reference<beans::XPropertySet> xProps(xSheet, uno::UNO_QUERY_THROW);
+    uno::Any aAny = xProps->getPropertyValue("ConditionalFormats");
+    uno::Reference<sheet::XConditionalFormats> xCondFormats;
+
+    CPPUNIT_ASSERT(aAny >>= xCondFormats);
+    CPPUNIT_ASSERT(xCondFormats.is());
+
+    CPPUNIT_ASSERT_EQUAL(sal_Int32(5), xCondFormats->getLength());
+
+    uno::Sequence<uno::Reference<sheet::XConditionalFormat>> xCondFormatSeq
+        = xCondFormats->getConditionalFormats();
+    CPPUNIT_ASSERT_EQUAL(sal_Int32(5), xCondFormatSeq.getLength());
+
+    int nRanges = 0;
+    for (sal_Int32 i = 0, n = xCondFormatSeq.getLength(); i < n; ++i)
+    {
+        CPPUNIT_ASSERT(xCondFormatSeq[i].is());
+
+        uno::Reference<sheet::XConditionalFormat> xCondFormat = xCondFormatSeq[i];
+        CPPUNIT_ASSERT(xCondFormat.is());
+
+        uno::Reference<beans::XPropertySet> xPropSet(xCondFormat, uno::UNO_QUERY_THROW);
+
+        aAny = xPropSet->getPropertyValue("Range");
+        uno::Reference<sheet::XSheetCellRanges> xCellRanges;
+        CPPUNIT_ASSERT(aAny >>= xCellRanges);
+        CPPUNIT_ASSERT(xCellRanges.is());
+
+        uno::Sequence<table::CellRangeAddress> aRanges = xCellRanges->getRangeAddresses();
+        CPPUNIT_ASSERT_GREATEREQUAL(sal_Int32(1), aRanges.getLength());
+
+        table::CellRangeAddress aRange0 = aRanges[0];
+        CPPUNIT_ASSERT_EQUAL(sal_Int16(0), aRange0.Sheet);
+        CPPUNIT_ASSERT_EQUAL(aRange0.StartColumn, aRange0.EndColumn);
+
+        table::CellRangeAddress aRange1;
+
+        switch (aRange0.StartColumn)
+        {
+            case 3:
+                switch (aRange0.StartRow)
+                {
+                    case 0: // D1:D2,D5::D8
+                        nRanges++;
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(1), aRange0.EndRow);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(2), aRanges.getLength());
+                        aRange1 = aRanges[1];
+                        CPPUNIT_ASSERT_EQUAL(sal_Int16(0), aRange1.Sheet);
+                        CPPUNIT_ASSERT_EQUAL(aRange1.StartColumn, aRange1.EndColumn);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(3), aRange1.StartColumn);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(4), aRange1.StartRow);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(7), aRange1.EndRow);
+                        break;
+                    default:
+                        CPPUNIT_FAIL("Unexpected range in column D");
+                }
+                break;
+            case 5:
+                switch (aRange0.StartRow)
+                {
+                    case 0: // F1:F2
+                        nRanges++;
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(1), aRange0.EndRow);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(1), aRanges.getLength());
+                        break;
+                    case 2: // F3
+                        nRanges++;
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(2), aRange0.EndRow);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(1), aRanges.getLength());
+                        break;
+                    case 3: // F4
+                        nRanges++;
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(3), aRange0.EndRow);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(1), aRanges.getLength());
+                        break;
+                    case 4: // F5
+                        nRanges++;
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(4), aRange0.EndRow);
+                        CPPUNIT_ASSERT_EQUAL(sal_Int32(1), aRanges.getLength());
+                        break;
+                    default:
+                        CPPUNIT_FAIL("Unexpected range in column F");
+                }
+                break;
+            default:
+                CPPUNIT_FAIL("Unexpected range");
+        }
+    }
+
+    CPPUNIT_ASSERT_EQUAL(5, nRanges);
+
+    closeDocument(mxComponent);
+    mxComponent.clear();
+}
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ScCondFormatMergeTest);
+
+CPPUNIT_PLUGIN_IMPLEMENT();
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

sc/source/core/data/conditio.cxx

@@ -650,6 +650,25 @@ bool ScConditionEntry::operator== ( const ScConditionEntry& r ) const
     return bEq;
 }
 
+bool ScConditionEntry::EqualIgnoringSrcPos( const ScConditionEntry& r ) const
+{
+    bool bEq = (eOp == r.eOp && nOptions == r.nOptions &&
+                lcl_IsEqual( pFormula1, r.pFormula1 ) &&
+                lcl_IsEqual( pFormula2, r.pFormula2 ));
+    if (bEq)
+    {
+        // Here, ignore the aSrcPoses and aSrcStrings
+
+        // If not formulas, compare values
+        if ( !pFormula1 && ( nVal1 != r.nVal1 || aStrVal1 != r.aStrVal1 || bIsStr1 != r.bIsStr1 ) )
+            bEq = false;
+        if ( !pFormula2 && ( nVal2 != r.nVal2 || aStrVal2 != r.aStrVal2 || bIsStr2 != r.bIsStr2 ) )
+            bEq = false;
+    }
+
+    return bEq;
+}
+
 void ScConditionEntry::Interpret( const ScAddress& rPos )
 {
     // Create formula cells

sc/source/filter/xml/xmlcondformat.hxx

@@ -10,6 +10,7 @@
 #ifndef INCLUDED_SC_SOURCE_FILTER_XML_XMLCONDFORMAT_HXX
 #define INCLUDED_SC_SOURCE_FILTER_XML_XMLCONDFORMAT_HXX
 
+#include <array>
 #include <memory>
 #include <xmloff/xmlictxt.hxx>
 #include "xmlimprt.hxx"
@@ -25,6 +26,21 @@ struct ScIconSetFormatData;
 
 class ScXMLConditionalFormatsContext : public ScXMLImportContext
 {
+private:
+    struct CacheEntry
+    {
+        ScConditionalFormat* mpFormat = nullptr;
+        bool mbSingleRelativeReference;
+        std::unique_ptr<const ScTokenArray> mpTokens;
+        sal_Int64 mnAge = SAL_MAX_INT64;
+    };
+
+    struct CondFormatData
+    {
+        ScConditionalFormat* mpFormat;
+        SCTAB mnTab;
+    };
+
 public:
     ScXMLConditionalFormatsContext( ScXMLImport& rImport );
 
@@ -32,13 +48,18 @@ public:
         sal_Int32 nElement, const css::uno::Reference< css::xml::sax::XFastAttributeList >& xAttrList ) override;
 
     virtual void SAL_CALL endFastElement( sal_Int32 nElement ) override;
+
+    std::array<CacheEntry, 4> maCache;
+
+    std::vector<CondFormatData> mvCondFormatData;
 };
 
 class ScXMLConditionalFormatContext : public ScXMLImportContext
 {
 public:
     ScXMLConditionalFormatContext( ScXMLImport& rImport,
-                        const rtl::Reference<sax_fastparser::FastAttributeList>& rAttrList );
+                                   const rtl::Reference<sax_fastparser::FastAttributeList>& rAttrList,
+                                   ScXMLConditionalFormatsContext& rParent );
 
     virtual ~ScXMLConditionalFormatContext() override;
 
@@ -50,6 +71,8 @@ private:
 
     std::unique_ptr<ScConditionalFormat> mxFormat;
     ScRangeList maRange;
+
+    ScXMLConditionalFormatsContext& mrParent;
 };
 
 class ScXMLColorScaleFormatContext : public ScXMLImportContext