Added XML data provider

Change-Id: Ib5727912977eb79cdf1f84bf874919beafc693eb
Reviewed-on: https://gerrit.libreoffice.org/56356
Tested-by: Jenkins
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

sc/Library_sc.mk

@@ -399,6 +399,7 @@ $(eval $(call gb_Library_add_exception_objects,sc,\
     sc/source/ui/dataprovider/dataprovider \
     sc/source/ui/dataprovider/datatransformation \
     sc/source/ui/dataprovider/htmldataprovider \
+    sc/source/ui/dataprovider/xmldataprovider \
     sc/source/ui/dbgui/asciiopt \
     sc/source/ui/dbgui/consdlg \
     sc/source/ui/dbgui/csvcontrol \

sc/inc/datamapper.hxx

@@ -16,6 +16,8 @@
 
 #include <rtl/ustring.hxx>
 
+#include "orcusxml.hxx"
+
 class ScDocument;
 class ScDBData;
 
@@ -62,6 +64,8 @@ private:
      */
     OUString maID;
 
+    ScOrcusImportXMLParam maParam;
+
     double mnUpdateFrequency;
 
     std::shared_ptr<DataProvider> mpDataProvider;
@@ -83,10 +87,12 @@ public:
     void setID(const OUString& rID);
     void setURL(const OUString& rURL);
     void setProvider(const OUString& rProvider);
+    void setXMLImportParam(const ScOrcusImportXMLParam& rParam);
 
     const OUString& getURL() const;
     const OUString& getProvider() const;
     const OUString& getID() const;
+    const ScOrcusImportXMLParam& getXMLImportParam() const;
     double getUpdateFrequency() const;
     OUString getDBName() const;
     void setDBData(const OUString& rDBName);

sc/qa/unit/data/dataprovider/xml/test1.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<bookstore>
+  <book>
+    <title>1</title>
+    <author>test1</author>
+  </book>
+  <book>
+    <title>2</title>
+    <author>test2</author>
+  </book>
+    <book>
+    <title>3</title>
+    <author>test3</author>
+  </book>
+    <book>
+    <title>4</title>
+    <author>test4</author>
+  </book>
+</bookstore>
\ No newline at end of file

sc/qa/unit/dataproviders_test.cxx

@@ -14,6 +14,7 @@
 #include <address.hxx>
 #include <dataprovider.hxx>
 #include <vcl/scheduler.hxx>
+#include <orcusxml.hxx>
 
 #include <memory>
 
@@ -29,11 +30,13 @@ public:
     void testCSVImport();
     void testDataLargerThanDB();
     void testHTMLImport();
+    void testXMLImport();
 
     CPPUNIT_TEST_SUITE(ScDataProvidersTest);
     CPPUNIT_TEST(testCSVImport);
     CPPUNIT_TEST(testDataLargerThanDB);
     CPPUNIT_TEST(testHTMLImport);
+    CPPUNIT_TEST(testXMLImport);
     CPPUNIT_TEST_SUITE_END();
 
 private:
@@ -137,6 +140,47 @@ void ScDataProvidersTest::testHTMLImport()
     }
 }
 
+void ScDataProvidersTest::testXMLImport()
+{
+    ScDBData* pDBData = new ScDBData("testDB", 0, 0, 0, 10, 10);
+    bool bInserted = m_pDoc->GetDBCollection()->getNamedDBs().insert(pDBData);
+    CPPUNIT_ASSERT(bInserted);
+
+    OUString aFileURL;
+    ScOrcusImportXMLParam aParam;
+
+    ScOrcusImportXMLParam::RangeLink aRangeLink;
+    aRangeLink.maPos = ScAddress(0,0,0);
+    aRangeLink.maFieldPaths.push_back("/bookstore/book/title");
+    aRangeLink.maFieldPaths.push_back("/bookstore/book/author");
+    aParam.maRangeLinks.push_back(aRangeLink);
+
+    createFileURL("test1.", "xml", aFileURL);
+    sc::ExternalDataSource aDataSource(aFileURL, "org.libreoffice.calc.xml", m_pDoc);
+    aDataSource.setDBData("testDB");
+    aDataSource.setXMLImportParam(aParam);
+
+
+    m_pDoc->GetExternalDataMapper().insertDataSource(aDataSource);
+    auto& rDataSources = m_pDoc->GetExternalDataMapper().getDataSources();
+    CPPUNIT_ASSERT(!rDataSources.empty());
+
+    rDataSources[0].refresh(m_pDoc, true);
+    Scheduler::ProcessEventsToIdle();
+
+    CPPUNIT_ASSERT_EQUAL(OUString("title"), m_pDoc->GetString(0, 0, 0));
+    CPPUNIT_ASSERT_EQUAL(OUString("author"), m_pDoc->GetString(1, 0, 0));
+    CPPUNIT_ASSERT_EQUAL(1.0, m_pDoc->GetValue(0, 1, 0));
+    CPPUNIT_ASSERT_EQUAL(OUString("test1"), m_pDoc->GetString(1, 1, 0));
+    CPPUNIT_ASSERT_EQUAL(2.0, m_pDoc->GetValue(0, 2, 0));
+    CPPUNIT_ASSERT_EQUAL(OUString("test2"), m_pDoc->GetString(1, 2, 0));
+    CPPUNIT_ASSERT_EQUAL(3.0, m_pDoc->GetValue(0, 3, 0));
+    CPPUNIT_ASSERT_EQUAL(OUString("test3"), m_pDoc->GetString(1, 3, 0));
+    CPPUNIT_ASSERT_EQUAL(4.0, m_pDoc->GetValue(0, 4, 0));
+    CPPUNIT_ASSERT_EQUAL(OUString("test4"), m_pDoc->GetString(1, 4, 0));
+}
+
+
 ScDataProvidersTest::ScDataProvidersTest() :
     ScBootstrapFixture( "sc/qa/unit/data/dataprovider" ),
     m_pDoc(nullptr)

sc/source/ui/dataprovider/dataprovider.cxx

@@ -15,6 +15,7 @@
 #include <rtl/strbuf.hxx>
 
 #include "htmldataprovider.hxx"
+#include "xmldataprovider.hxx"
 #include <datatransformation.hxx>
 
 using namespace com::sun::star;
@@ -74,6 +75,13 @@ void ExternalDataSource::setID(const OUString& rID)
     maID = rID;
 }
 
+void ExternalDataSource::setXMLImportParam(const ScOrcusImportXMLParam& rParam)
+{
+    maParam = rParam;
+}
+
+
+
 void ExternalDataSource::setURL(const OUString& rURL)
 {
     maURL = rURL;
@@ -100,6 +108,11 @@ const OUString& ExternalDataSource::getID() const
     return maID;
 }
 
+const ScOrcusImportXMLParam& ExternalDataSource::getXMLImportParam() const
+{
+    return maParam;
+}
+
 OUString ExternalDataSource::getDBName() const
 {
     if (mpDBDataManager)
@@ -274,6 +287,8 @@ std::shared_ptr<DataProvider> DataProviderFactory::getDataProvider(ScDocument* p
             return std::shared_ptr<DataProvider>(new CSVDataProvider(pDoc, rDataSource));
         else if (rDataProvider == "org.libreoffice.calc.html")
             return std::shared_ptr<DataProvider>(new HTMLDataProvider(pDoc, rDataSource));
+        else if (rDataProvider == "org.libreoffice.calc.xml")
+            return std::shared_ptr<DataProvider>(new XMLDataProvider(pDoc, rDataSource));
     }
     else
     {
@@ -289,6 +304,7 @@ std::vector<OUString> DataProviderFactory::getDataProviders()
     std::vector<OUString> aDataProviders;
     aDataProviders.emplace_back("org.libreoffice.calc.csv");
     aDataProviders.emplace_back("org.libreoffice.calc.html");
+    aDataProviders.emplace_back("org.libreoffice.calc.xml");
 
     return aDataProviders;
 }

sc/source/ui/dataprovider/xmldataprovider.cxx

+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include "xmldataprovider.hxx"
+#include <datatransformation.hxx>
+#include <salhelper/thread.hxx>
+#include <comphelper/string.hxx>
+#include <filter.hxx>
+#include <document.hxx>
+#include <orcusfilters.hxx>
+
+using namespace com::sun::star;
+
+namespace sc
+{
+class XMLFetchThread : public salhelper::Thread
+{
+    ScDocument& mrDocument;
+    OUString maURL;
+    OUString maID;
+    ScOrcusImportXMLParam maParam;
+    std::unique_ptr<ScOrcusXMLContext> mpXMLContext;
+    const std::vector<std::shared_ptr<sc::DataTransformation>> maDataTransformations;
+    std::function<void()> maImportFinishedHdl;
+
+public:
+    XMLFetchThread(ScDocument& rDoc, const OUString&, const ScOrcusImportXMLParam& rParam,
+                   const OUString& rID, std::function<void()> aImportFinishedHdl,
+                   const std::vector<std::shared_ptr<sc::DataTransformation>>& rTransformations);
+    virtual void execute() override;
+};
+
+XMLFetchThread::XMLFetchThread(
+    ScDocument& rDoc, const OUString& rURL, const ScOrcusImportXMLParam& rParam,
+    const OUString& rID, std::function<void()> aImportFinishedHdl,
+    const std::vector<std::shared_ptr<sc::DataTransformation>>& rTransformations)
+    : salhelper::Thread("XML Fetch Thread")
+    , mrDocument(rDoc)
+    , maURL(rURL)
+    , maID(rID)
+    , maParam(rParam)
+    , maDataTransformations(rTransformations)
+    , maImportFinishedHdl(aImportFinishedHdl)
+{
+}
+
+void XMLFetchThread::execute()
+{
+    ScOrcusFilters* pOrcus = ScFormatFilter::Get().GetOrcusFilters();
+    if (!pOrcus)
+        return;
+
+    mpXMLContext.reset(pOrcus->createXMLContext(mrDocument, maURL));
+    if (!mpXMLContext)
+        return;
+
+    if (!maID.isEmpty())
+    {
+        ScOrcusImportXMLParam::RangeLink aRangeLink;
+        aRangeLink.maPos = ScAddress(0, 0, 0);
+        aRangeLink.maFieldPaths.push_back(OUStringToOString(maID, RTL_TEXTENCODING_UTF8));
+        maParam.maRangeLinks.clear();
+        maParam.maRangeLinks.push_back(aRangeLink);
+    }
+    // Do the import.
+    mpXMLContext->importXML(maParam);
+
+    for (auto& itr : maDataTransformations)
+    {
+        itr->Transform(mrDocument);
+    }
+
+    SolarMutexGuard aGuard;
+    maImportFinishedHdl();
+}
+
+XMLDataProvider::XMLDataProvider(ScDocument* pDoc, sc::ExternalDataSource& rDataSource)
+    : DataProvider(rDataSource)
+    , mpDocument(pDoc)
+{
+}
+
+XMLDataProvider::~XMLDataProvider()
+{
+    if (mxXMLFetchThread.is())
+    {
+        SolarMutexReleaser aReleaser;
+        mxXMLFetchThread->join();
+    }
+}
+
+void XMLDataProvider::Import()
+{
+    // already importing data
+    if (mpDoc)
+        return;
+
+    mpDoc.reset(new ScDocument(SCDOCMODE_CLIP));
+    mpDoc->ResetClip(mpDocument, SCTAB(0));
+    mxXMLFetchThread = new XMLFetchThread(
+        *mpDoc, mrDataSource.getURL(), mrDataSource.getXMLImportParam(), mrDataSource.getID(),
+        std::bind(&XMLDataProvider::ImportFinished, this), mrDataSource.getDataTransformation());
+    mxXMLFetchThread->launch();
+
+    if (mbDeterministic)
+    {
+        SolarMutexReleaser aReleaser;
+        mxXMLFetchThread->join();
+    }
+}
+
+void XMLDataProvider::ImportFinished()
+{
+    mrDataSource.getDBManager()->WriteToDoc(*mpDoc);
+    mxXMLFetchThread.clear();
+    mpDoc.reset();
+}
+
+const OUString& XMLDataProvider::GetURL() const { return mrDataSource.getURL(); }
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

sc/source/ui/dataprovider/xmldataprovider.hxx

+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#ifndef INCLUDED_SC_SOURCE_UI_DATAPROVIDER_XMLDATAPROVIDER_HXX
+#define INCLUDED_SC_SOURCE_UI_DATAPROVIDER_XMLDATAPROVIDER_HXX
+
+#include <dataprovider.hxx>
+
+namespace sc
+{
+class XMLFetchThread;
+class DataTransformation;
+
+class XMLDataProvider : public DataProvider
+{
+private:
+    ScDocument* mpDocument;
+    rtl::Reference<XMLFetchThread> mxXMLFetchThread;
+    ScDocumentUniquePtr mpDoc;
+
+public:
+    XMLDataProvider(ScDocument* pDoc, sc::ExternalDataSource& rDataSource);
+    virtual ~XMLDataProvider() override;
+
+    virtual void Import() override;
+
+    virtual const OUString& GetURL() const override;
+
+    void ImportFinished();
+};
+}
+
+#endif
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */