tdf#73691 Implement MSWord's Alt-X: toggle unicode notation

-toggles between characters and their unicode notation
-sets Alt-X as a global keyboard accelerator
-handles all of the unicode planes
-intelligently handles combining characters
-if text is selected, limits the input to that text
-implemented in Writer, Draw, Impress

Change-Id: Idcd8e7f0a4f1b81fa7f5f3200c76be19472ffa37
Reviewed-on: https://gerrit.libreoffice.org/17535Tested-by: Samuel Mehrbrodt <s.mehrbrodt@gmail.com>
Reviewed-by: Miklos Vajna <vmiklos@collabora.co.uk>

i18nutil/source/utility/unicode.cxx

@@ -27,6 +27,8 @@
 #include <sal/log.hxx>
 #include <unicode/numfmt.h>
 #include "unicode_data.h"
+#include <com/sun/star/i18n/UnicodeType.hpp>
+#include <rtl/character.hxx>
 
 // Workaround for glibc braindamage:
 // glibc 2.4's langinfo.h does "#define CURRENCY_SYMBOL __CURRENCY_SYMBOL"
@@ -998,4 +1000,253 @@ OUString SAL_CALL unicode::formatPercent(double dNumber,
     return aRet;
 }
 
+ToggleUnicodeCodepoint::ToggleUnicodeCodepoint ()
+{
+    maInput = OUStringBuffer();
+    maOutput = OUStringBuffer();
+    maUtf16 = OUStringBuffer();
+    maCombining = OUStringBuffer();
+}
+
+bool ToggleUnicodeCodepoint::AllowMoreInput(sal_Unicode uChar)
+{
+    //arbitrarily chosen maximum length allowed - normal max usage would be around 30.
+    if( maInput.getLength() > 255 )
+        mbAllowMoreChars = false;
+
+    if( !mbAllowMoreChars )
+        return false;
+
+    bool bPreventNonHex = false;
+    if( maInput.indexOf("U+") != -1 )
+        bPreventNonHex = true;
+
+    switch ( unicode::getUnicodeType(uChar) )
+    {
+        case ::com::sun::star::i18n::UnicodeType::SURROGATE:
+            if( bPreventNonHex )
+            {
+                mbAllowMoreChars = false;
+                return false;
+            }
+
+            if( rtl::isLowSurrogate(uChar) && maUtf16.isEmpty() && maInput.isEmpty()  )
+            {
+                maUtf16.append(uChar);
+                return true;
+            }
+            if( rtl::isHighSurrogate(uChar) && maInput.isEmpty() )
+                maUtf16.insert(0, uChar );
+            //end of hex strings, or unexpected order of high/low, so don't accept more
+            if( !maUtf16.isEmpty() )
+                maInput.append(maUtf16);
+            if( !maCombining.isEmpty() )
+                maInput.append(maCombining);
+            mbAllowMoreChars = false;
+            break;
+
+        case ::com::sun::star::i18n::UnicodeType::NON_SPACING_MARK:
+        case ::com::sun::star::i18n::UnicodeType::COMBINING_SPACING_MARK:
+            if( bPreventNonHex )
+            {
+                mbAllowMoreChars = false;
+                return false;
+            }
+
+            //extreme edge case: already invalid high/low surrogates with preceding combining chars, and now an extra combining mark.
+            if( !maUtf16.isEmpty() )
+            {
+                maInput = maUtf16;
+                if( !maCombining.isEmpty() )
+                    maInput.append(maCombining);
+                mbAllowMoreChars = false;
+                return false;
+            }
+            maCombining.insert(0, uChar);
+            break;
+
+        default:
+            //extreme edge case: already invalid high/low surrogates with preceding combining chars, and now an extra character.
+            if( !maUtf16.isEmpty() )
+            {
+                maInput = maUtf16;
+                if( !maCombining.isEmpty() )
+                    maInput.append(maCombining);
+                mbAllowMoreChars = false;
+                return false;
+            }
+
+            if( !maCombining.isEmpty() )
+            {
+                maCombining.insert(0, uChar);
+                maInput = maCombining;
+                mbAllowMoreChars = false;
+                return false;
+            }
+
+            switch( uChar )
+            {
+                case 'u':
+                case 'U':
+                    // U+ notation found.  Continue looking for another one.
+                    if( mbRequiresU )
+                    {
+                        mbRequiresU = false;
+                        maInput.insert(0,"U+");
+                    }
+                    // treat as a normal character
+                    else
+                    {
+                        mbAllowMoreChars = false;
+                        if( !bPreventNonHex )
+                            maInput.insertUtf32(0, uChar);
+                    }
+                    break;
+                case '+':
+                    // + already found: skip when not U, or edge case of +U+xxxx
+                    if( mbRequiresU || (maInput.indexOf("U+") == 0) )
+                        mbAllowMoreChars = false;
+                    // hex chars followed by '+' - now require a 'U'
+                    else if ( !maInput.isEmpty() )
+                        mbRequiresU = true;
+                    // treat as a normal character
+                    else
+                    {
+                        mbAllowMoreChars = false;
+                        if( !bPreventNonHex )
+                            maInput.insertUtf32(0, uChar);
+                    }
+                    break;
+                case 0:
+                    mbAllowMoreChars = false;
+                    break;
+                default:
+                    // + already found. Since not U, cancel further input
+                    if( mbRequiresU )
+                        mbAllowMoreChars = false;
+                    // maximum digits per notation is 8: only one notation
+                    else if( maInput.indexOf("U+") == -1 && maInput.getLength() == 8 )
+                        mbAllowMoreChars = false;
+                    // maximum digits per notation is 8: previous notation found
+                    else if( maInput.indexOf("U+") == 8 )
+                        mbAllowMoreChars = false;
+                    // a hex character. Add to string.
+                    else if( isxdigit(uChar) )
+                    {
+                        mbIsHexString = true;
+                        maInput.insertUtf32(0, uChar);
+                    }
+                    // not a hex character: stop input. keep if it is the first input provided
+                    else
+                    {
+                        mbAllowMoreChars = false;
+                        if( maInput.isEmpty() )
+                            maInput.insertUtf32(0, uChar);
+                    }
+            }
+    }
+    return mbAllowMoreChars;
+}
+
+OUString ToggleUnicodeCodepoint::StringToReplace()
+{
+    if( maInput.isEmpty() )
+    {
+        //edge case - input finished with incomplete low surrogate or combining characters without a base
+        if( mbAllowMoreChars )
+        {
+            if( !maUtf16.isEmpty() )
+                maInput = maUtf16;
+            if( !maCombining.isEmpty() )
+                maInput.append(maCombining);
+        }
+        return maInput.toString();
+    }
+
+    if( !mbIsHexString )
+        return maInput.toString();
+
+    //this function potentially modifies the input string.  Prevent addition of further characters
+    mbAllowMoreChars = false;
+
+    //validate unicode notation.
+    OUStringBuffer sIn;
+    sal_uInt32 nUnicode = 0;
+    sal_Int32 nUPlus = maInput.indexOf("U+");
+    //if U+ notation used, strip off all extra chars added not in U+ notation
+    if( nUPlus != -1 )
+    {
+        maInput = maInput.copy(nUPlus);
+        sIn = maInput.copy(2);
+        nUPlus = sIn.indexOf("U+");
+    }
+    else
+        sIn = maInput;
+    while( nUPlus != -1 )
+    {
+        nUnicode = sIn.copy(0, nUPlus).toString().toUInt32(16);
+        //strip out all null or invalid Unicode values
+        if( !nUnicode || nUnicode > 0x10ffff )
+            maInput = sIn.copy(nUPlus);
+        sIn = sIn.copy(nUPlus+2);
+        nUPlus =  sIn.indexOf("U+");
+    }
+
+    nUnicode = sIn.toString().toUInt32(16);
+    if( !nUnicode || nUnicode > 0x10ffff )
+       maInput.truncate(0).append( sIn[sIn.getLength()-1] );
+    return maInput.toString();
+}
+
+sal_uInt32 ToggleUnicodeCodepoint::CharsToDelete()
+{
+    OUString sIn = StringToReplace();
+    sal_Int32 nPos = 0;
+    sal_uInt32 counter = 0;
+    while( nPos < sIn.getLength() )
+    {
+        sIn.iterateCodePoints(&nPos,1);
+        ++counter;
+    }
+    return counter;
+}
+
+OUString ToggleUnicodeCodepoint::ReplacementString()
+{
+    OUString sIn = StringToReplace();
+    maOutput = "";
+    sal_Int32 nUPlus = sIn.indexOf("U+");
+    // convert from hex notation to glyph
+    if( nUPlus != -1 || (sIn.getLength() > 1 && mbIsHexString) )
+    {
+        sal_uInt32 nUnicode = 0;
+        if( nUPlus == 0)
+        {
+            sIn = sIn.copy(2);
+            nUPlus = sIn.indexOf("U+");
+        }
+        while( nUPlus > 0 )
+        {
+            nUnicode = sIn.copy(0, nUPlus).toUInt32(16);
+            maOutput.appendUtf32( nUnicode );
+
+            sIn = sIn.copy(nUPlus+2);
+            nUPlus = sIn.indexOf("U+");
+        }
+        nUnicode = sIn.toUInt32(16);
+        maOutput.appendUtf32( nUnicode );
+    }
+    // convert from glyph to hex notation
+    else
+    {
+        sal_Int32 nPos = 0;
+        while( nPos < sIn.getLength() )
+        {
+            maOutput.append( "U+" );
+            maOutput.append( OUString::number(sIn.iterateCodePoints(&nPos,1),16) );
+        }
+    }
+    return maOutput.toString();
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

include/i18nutil/unicode.hxx

@@ -21,6 +21,7 @@
 
 #include <com/sun/star/i18n/UnicodeScript.hpp>
 #include <sal/types.h>
+#include <rtl/ustrbuf.hxx>
 #include <unicode/uscript.h>
 #include <i18nutil/i18nutildllapi.h>
 
@@ -58,6 +59,51 @@ public:
         const LanguageTag &rLangTag);
 };
 
+/*
+    Toggle between a character and its Unicode Notation.
+        -implements the concept found in Microsoft Word's Alt-X
+        -accepts sequences of up to 8 hex characters and converts into the corresponding Unicode Character
+            -example:  0000A78c   or   2bc
+        -accepts sequences of up to 256 characters in Unicode notation
+            -example:  U+00000065u+0331u+308
+        -handles complex characters (with combining elements) and the all of the Unicode planes.
+*/
+class I18NUTIL_DLLPUBLIC ToggleUnicodeCodepoint
+{
+private:
+    OUStringBuffer maInput;
+    OUStringBuffer maOutput;
+    OUStringBuffer maUtf16;
+    OUStringBuffer maCombining;
+    bool mbAllowMoreChars = true;
+    bool mbRequiresU = false;
+    bool mbIsHexString = false;
+
+public:
+    ToggleUnicodeCodepoint();
+
+    /**
+    Build an input string of valid UTF16 units to toggle.
+        -do not call the other functions until the input process is complete
+        -build string from Right to Left.  (Start from the character to the left of the cursor: move left.)
+    */
+    bool AllowMoreInput(sal_Unicode uChar);
+
+    /**
+    Validates (and potentially modifies) the input string.
+        -all non-input functions must use this function to first to validate the input string
+        -additional input may be prevented after this function is called
+    */
+    OUString StringToReplace();
+    OUString ReplacementString();
+
+    /**
+    While sInput.getLength() returns the number of utf16 units to delete,
+        this function returns the number of "characters" to delete - potentially a smaller number
+    */
+    sal_uInt32 CharsToDelete();
+};
+
 #endif
 
 

include/sfx2/sfxcommands.h

@@ -46,6 +46,7 @@
 #define CMD_SID_PRINTPREVIEW                        ".uno:PrintPreview"
 #define CMD_SID_RELOAD                              ".uno:Reload"
 #define CMD_SID_BASICIDE_RENAMECURRENT              ".uno:RenameCurrent"
+#define CMD_SID_UNICODE_NOTATION_TOGGLE             ".uno:UnicodeNotationToggle"
 
 #endif
 

include/sfx2/sfxsids.hrc

@@ -415,6 +415,7 @@
 #define SID_OBJECTMENU3                     (SID_SFX_START + 783)
 #define SID_OBJECTMENU_LAST                 SID_OBJECTMENU3
 #define SID_FORMATMENUSTATE                 (SID_SFX_START + 791)
+#define SID_UNICODE_NOTATION_TOGGLE         (SID_SFX_START + 792)
 
 // default-ids for macros
 #define SID_RECORDING_FLOATWINDOW           (SID_SFX_START + 800)

officecfg/registry/data/org/openoffice/Office/Accelerators.xcu

@@ -208,6 +208,12 @@
           <value xml:lang="en-US">.uno:Cut</value>
         </prop>
       </node>
+      <node oor:name="X_MOD2" oor:op="replace">
+        <prop oor:name="Command">
+          <value xml:lang="x-no-translate">I10N SHORTCUTS - NO TRANSLATE</value>
+          <value xml:lang="en-US">.uno:UnicodeNotationToggle</value>
+        </prop>
+      </node>
       <node oor:name="Y_SHIFT_MOD1" oor:op="replace">
         <prop oor:name="Command">
           <value xml:lang="x-no-translate">I10N SHORTCUTS - NO TRANSLATE</value>

officecfg/registry/data/org/openoffice/Office/UI/GenericCommands.xcu

@@ -33,6 +33,11 @@
           <value xml:lang="en-US">New Presentation</value>
         </prop>
       </node>
+      <node oor:name=".uno:UnicodeNotationToggle" oor:op="replace">
+        <prop oor:name="Label" oor:type="xs:string">
+          <value xml:lang="en-US">Toggle Unicode Notation</value>
+        </prop>
+      </node>
       <node oor:name=".uno:FontworkGalleryFloater" oor:op="replace">
         <prop oor:name="Label" oor:type="xs:string">
           <value xml:lang="en-US">Fontwork Gallery...</value>

sd/sdi/_drvwsh.sdi

@@ -96,6 +96,10 @@ interface DrawView
         ExecMethod = FuSupport ;
         StateMethod = GetMenuState ;
     ]
+    SID_UNICODE_NOTATION_TOGGLE
+    [
+        ExecMethod = FuSupport;
+    ]
     SID_PASTE_UNFORMATTED
     [
         ExecMethod = FuSupport ;

sd/sdi/sdraw.sdi

@@ -7208,4 +7208,27 @@ SfxVoidItem MovePageLast SID_MOVE_PAGE_LAST
     ToolBoxConfig = TRUE,
     GroupId = GID_MODIFY;
 ]
+SfxVoidItem UnicodeNotationToggle SID_UNICODE_NOTATION_TOGGLE
+()
+[
+    /* flags: */
+    AutoUpdate = FALSE,
+    Cachable = Cachable,
+    FastCall = FALSE,
+    HasCoreId = FALSE,
+    HasDialog = FALSE,
+    ReadOnlyDoc = FALSE,
+    Toggle = FALSE,
+    Container = FALSE,
+    RecordAbsolute = FALSE,
+    RecordPerSet;
+    Synchron;
+
+    /* config: */
+    AccelConfig = TRUE,
+    MenuConfig = FALSE,
+    StatusBarConfig = FALSE,
+    ToolBoxConfig = FALSE,
+    GroupId = GID_OPTIONS;
+]
 

sd/source/ui/view/drviewse.cxx

@@ -22,6 +22,7 @@
 #include <com/sun/star/beans/XPropertySet.hpp>
 #include <com/sun/star/i18n/TransliterationModules.hpp>
 #include <com/sun/star/i18n/TransliterationModulesExtra.hpp>
+#include <i18nutil/unicode.hxx>
 #include <com/sun/star/beans/PropertyValue.hpp>
 #include <com/sun/star/uno/Any.hxx>
 
@@ -837,6 +838,47 @@ void DrawViewShell::FuSupport(SfxRequest& rReq)
         }
         break;
 
+        case SID_UNICODE_NOTATION_TOGGLE:
+        {
+            if ( mpDrawView->IsTextEdit() )
+            {
+                OutlinerView* pOLV = mpDrawView->GetTextEditOutlinerView();
+                if (pOLV)
+                {
+                    OUString sInput = pOLV->GetSurroundingText();
+                    ESelection aSel( pOLV->GetSelection() );
+                    if ( aSel.nStartPos > aSel.nEndPos )
+                        aSel.nEndPos = aSel.nStartPos;
+
+                    //calculate a valid end-position by reading logical characters
+                    sal_Int32 nUtf16Pos=0;
+                    while( (nUtf16Pos < sInput.getLength()) && (nUtf16Pos < aSel.nEndPos) )
+                    {
+                        sInput.iterateCodePoints(&nUtf16Pos);
+                        //The mouse can set the cursor in the middle of a multi-unit character,
+                        //    so reset to the proper end of the logical characters
+                        if( nUtf16Pos > aSel.nEndPos )
+                            aSel.nEndPos = nUtf16Pos;
+                    }
+
+                    ToggleUnicodeCodepoint aToggle = ToggleUnicodeCodepoint();
+                    while( nUtf16Pos && aToggle.AllowMoreInput( sInput[nUtf16Pos-1]) )
+                        --nUtf16Pos;
+                    OUString sReplacement = aToggle.ReplacementString();
+                    if( !sReplacement.isEmpty() )
+                    {
+                        OUString sStringToReplace = aToggle.StringToReplace();
+                        mpDrawView->BegUndo(sStringToReplace +"->"+ sReplacement);
+                        aSel.nStartPos = aSel.nEndPos - sStringToReplace.getLength();
+                        pOLV->SetSelection( aSel );
+                        pOLV->InsertText(sReplacement, true);
+                        mpDrawView->EndUndo();
+                    }
+                }
+            }
+        }
+        break;
+
         case SID_PASTE_UNFORMATTED:
         {
             WaitObject aWait( GetActiveWindow() );

sw/qa/extras/uiwriter/uiwriter.cxx

@@ -147,6 +147,7 @@ public:
     void testTdf90883TableBoxGetCoordinates();
     void testEmbeddedDataSource();
     void testUnoCursorPointer();
+    void testUnicodeNotationToggle();
     void testTextTableCellNames();
     void testShapeAnchorUndo();
     void testDde();
@@ -215,6 +216,7 @@ public:
     CPPUNIT_TEST(testTdf90883TableBoxGetCoordinates);
     CPPUNIT_TEST(testEmbeddedDataSource);
     CPPUNIT_TEST(testUnoCursorPointer);
+    CPPUNIT_TEST(testUnicodeNotationToggle);
     CPPUNIT_TEST(testTextTableCellNames);
     CPPUNIT_TEST(testShapeAnchorUndo);
     CPPUNIT_TEST(testDde);
@@ -2176,6 +2178,29 @@ void SwUiWriterTest::testDde()
     CPPUNIT_ASSERT(xField->getString().endsWith("asdf"));
 }
 
+void SwUiWriterTest::testUnicodeNotationToggle()
+{
+    SwDoc* pDoc = createDoc("unicodeAltX.odt");
+    SwWrtShell* pWrtShell = pDoc->GetDocShell()->GetWrtShell();
+    OUString sOriginalDocString;
+    OUString sDocString;
+    OUString sExpectedString;
+    uno::Sequence<beans::PropertyValue> aPropertyValues;
+
+    pWrtShell->EndPara();
+    sOriginalDocString = pWrtShell->GetCrsr()->GetNode().GetTextNode()->GetText();
+    CPPUNIT_ASSERT( sOriginalDocString.equals("uU+2b") );
+
+    lcl_dispatchCommand(mxComponent, ".uno:UnicodeNotationToggle", aPropertyValues);
+    sExpectedString = "u+";
+    sDocString = pWrtShell->GetCrsr()->GetNode().GetTextNode()->GetText();
+    CPPUNIT_ASSERT( sDocString.equals(sExpectedString) );
+
+    lcl_dispatchCommand(mxComponent, ".uno:UnicodeNotationToggle", aPropertyValues);
+    sDocString = pWrtShell->GetCrsr()->GetNode().GetTextNode()->GetText();
+    CPPUNIT_ASSERT( sDocString.equals(sOriginalDocString) );
+}
+
 void SwUiWriterTest::testTdf89954()
 {
     SwDoc* pDoc = createDoc("tdf89954.odt");

sw/sdi/_textsh.sdi

@@ -1630,6 +1630,11 @@ interface BaseText
         DisableFlags="SW_DISABLE_ON_PROTECTED_CURSOR";  // e.g. disable for read-only documents
     ]
 
+    SID_UNICODE_NOTATION_TOGGLE
+    [
+        ExecMethod = Execute;
+    ]
+
     SID_THES
     [
         ExecMethod = Execute ;

sw/sdi/swriter.sdi

@@ -10249,3 +10249,28 @@ SfxVoidItem RemoveTextBox FN_REMOVE_TEXT_BOX
     ToolBoxConfig = TRUE,
     GroupId = GID_DRAWING;
 ]
+
+SfxVoidItem UnicodeNotationToggle SID_UNICODE_NOTATION_TOGGLE
+()
+[
+    /* flags: */
+    AutoUpdate = FALSE,
+    Cachable = Cachable,
+    FastCall = FALSE,
+    HasCoreId = FALSE,
+    HasDialog = FALSE,
+    ReadOnlyDoc = FALSE,
+    Toggle = FALSE,
+    Container = FALSE,
+    RecordAbsolute = FALSE,
+    RecordPerSet;
+    Asynchron;
+
+    /* config: */
+    AccelConfig = TRUE,
+    MenuConfig = FALSE,
+    StatusBarConfig = FALSE,
+    ToolBoxConfig = FALSE,
+    GroupId = GID_OPTIONS;
+]
+

sw/source/uibase/shells/textsh1.cxx

@@ -23,6 +23,7 @@
 #include <cmdid.h>
 #include <helpid.h>
 
+#include <i18nutil/unicode.hxx>
 #include <i18nlangtag/languagetag.hxx>
 #include <svl/languageoptions.hxx>
 #include <editeng/langitem.hxx>
@@ -113,7 +114,7 @@
 #include <tools/diagnose_ex.h>
 #include <svx/nbdtmgfact.hxx>
 #include <svx/nbdtmg.hxx>
-
+#include <SwRewriter.hxx>
 #include <svx/svdmodel.hxx>
 #include <svx/drawitem.hxx>
 #include <numrule.hxx>
@@ -287,6 +288,35 @@ void SwTextShell::Execute(SfxRequest &rReq)
         pArgs->GetItemState(GetPool().GetWhich(nSlot), false, &pItem);
     switch( nSlot )
     {
+        case SID_UNICODE_NOTATION_TOGGLE:
+        {
+            int nMaxUnits = 256;
+            if( rWrtSh.IsSelection() && !rWrtSh.IsMultiSelection() )
+                nMaxUnits = rWrtSh.GetSelText().getLength();
+
+            int index = 0;
+            ToggleUnicodeCodepoint aToggle = ToggleUnicodeCodepoint();
+            while( nMaxUnits-- && aToggle.AllowMoreInput(rWrtSh.GetChar(true, index-1)) )
+                --index;
+
+            OUString sReplacement = aToggle.ReplacementString();
+            if( !sReplacement.isEmpty() )
+            {
+                SwRewriter aRewriter;
+                aRewriter.AddRule( UndoArg1, aToggle.StringToReplace() );
+                aRewriter.AddRule( UndoArg2, "->" );
+                aRewriter.AddRule( UndoArg3, sReplacement );
+                rWrtSh.StartUndo(UNDO_REPLACE, &aRewriter);
+                rWrtSh.GetCrsr()->Normalize(false);
+                rWrtSh.ClearMark();
+                for( sal_uInt32 i=aToggle.CharsToDelete(); i > 0; --i )
+                    rWrtSh.DelLeft();
+                rWrtSh.Insert2( sReplacement );
+                rWrtSh.EndUndo(UNDO_REPLACE, &aRewriter);
+            }
+        }
+        break;
+
         case SID_LANGUAGE_STATUS:
         {
             // get the language