Skip to content

C
core
Kaydet (Commit) 4647e778 authored tarafından Mark Hung's avatar Mark Hung
Dosyalara gözat
Seçenekler

tdf#81129 Support reading non-BMP characters in HTML documents. 

1. Allow character entity ( &#nnnn; ) to exceed 0xffff in HTMLParser::ScanText()
2. Return a character as sal_uInt32 ( utf32 ) instead of sal_Unicode ( utf16 )
   from SvParser::GetNextChar().

Conflicts:
	sw/qa/extras/htmlexport/htmlexport.cxx

Change-Id: Ida455040970fae800f0f11471b27f53461fb78e4
Reviewed-on: https://gerrit.libreoffice.org/21152Tested-by: 's avatarJenkins <ci@libreoffice.org>
Reviewed-by: 's avatarMark Hung <marklh9@gmail.com>
üst 35966131
Hide whitespace changes
Inline Side-by-side
Showing with 90 additions and 38 deletions
include/rtl/character.hxx
Dosyayı görüntüle @ 4647e778
......@@ -222,6 +222,19 @@ sal_uInt32 const surrogatesLowLast = 0xDFFF;
}
/// @endcond
/** Check if a codepoint is accessible via utf16 per RFC3629
@param code A non-BMP Unicode code point.
@return True if the code is a valid codepoint.
@since LibreOffice 5.2
*/
inline bool isValidCodePoint( sal_uInt32 code)
{
return code <= 0x10FFFF;
}
/** Check for high surrogate.
@param code A Unicode code point.
......
include/svtools/svparser.hxx
Dosyayı görüntüle @ 4647e778
......@@ -59,7 +59,7 @@ protected:
rtl_TextEncoding eSrcEnc; // Source encoding
sal_uLong nNextChPos;
sal_Unicode nNextCh; // current character for the "lex"
sal_uInt32 nNextCh; // current character codepoint in UTF32 for the "lex"
bool bDownloadingFile : 1; // true: An external file is
......@@ -128,7 +128,7 @@ public:
inline void SetLineNr( sal_uLong nlNum ); // inline bottom
inline void SetLinePos( sal_uLong nlPos ); // inline bottom
sal_Unicode GetNextChar();
sal_uInt32 GetNextChar(); // Return next Unicode codepoint in UTF32.
void RereadLookahead();
inline bool IsParserWorking() const { return SVPAR_WORKING == eState; }
......
svtools/source/svhtml/parhtml.cxx
Dosyayı görüntüle @ 4647e778
......@@ -25,6 +25,7 @@
#include <tools/color.hxx>
#include <rtl/ustrbuf.hxx>
#include <rtl/strbuf.hxx>
#include <rtl/character.hxx>
#include <tools/tenccvt.hxx>
#include <tools/datetime.hxx>
......@@ -429,7 +430,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
OUStringBuffer sTmpBuffer( MAX_LEN );
bool bContinue = true;
bool bEqSignFound = false;
sal_Unicode cQuote = 0U;
sal_uInt32 cQuote = 0U;
while( bContinue && IsParserWorking() )
{
......@@ -445,7 +446,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
sal_uLong nStreamPos = rInput.Tell();
sal_uLong nLinePos = GetLinePos();
sal_Unicode cChar = 0U;
sal_uInt32 cChar = 0U;
if( '#' == (nNextCh = GetNextChar()) )
{
nNextCh = GetNextChar();
......@@ -460,10 +461,10 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
{
cChar = cChar * 16U +
( nNextCh <= '9'
? sal_Unicode( nNextCh - '0' )
? sal_uInt32( nNextCh - '0' )
: ( nNextCh <= 'F'
? sal_Unicode( nNextCh - 'A' + 10 )
: sal_Unicode( nNextCh - 'a' + 10 ) ) );
? sal_uInt32( nNextCh - 'A' + 10 )
: sal_uInt32( nNextCh - 'a' + 10 ) ) );
nNextCh = GetNextChar();
}
}
......@@ -471,7 +472,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
{
do
{
cChar = cChar * 10U + sal_Unicode( nNextCh - '0');
cChar = cChar * 10U + sal_uInt32( nNextCh - '0');
nNextCh = GetNextChar();
}
while( HTML_ISDIGIT(nNextCh) );
......@@ -500,6 +501,9 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
}
else
nNextCh = 0U;
if ( ! rtl::isValidCodePoint( cChar ) )
cChar = '?';
}
else if( HTML_ISALPHA( nNextCh ) )
{
......@@ -507,7 +511,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
sal_Int32 nPos = 0L;
do
{
sEntityBuffer.append( nNextCh );
sEntityBuffer.appendUtf32( nNextCh );
nPos++;
nNextCh = GetNextChar();
}
......@@ -637,7 +641,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
if( IsParserWorking() )
{
if( cChar )
sTmpBuffer.append( cChar );
sTmpBuffer.appendUtf32( cChar );
}
else if( SVPAR_PENDING==eState && '>'!=cBreak )
{
......@@ -661,7 +665,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
case '=':
if( '>'==cBreak && !cQuote )
bEqSignFound = true;
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
break;
case '\\':
......@@ -684,7 +688,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
else if( cQuote && (cQuote==nNextCh ) )
cQuote = 0U;
}
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
bEqSignFound = false;
break;
......@@ -695,14 +699,15 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
}
else
{
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
}
break;
case '<':
bEqSignFound = false;
if( '>'==cBreak )
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
else
bContinue = false; // break, String zusammen
break;
......@@ -725,7 +730,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
if( '>'==cBreak )
{
// cr/lf in tag is handled in _GetNextToken()
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
break;
}
else if( bReadListing || bReadXMP || bReadPRE || bReadTextArea )
......@@ -752,7 +757,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
nNextCh = ' ';
// no break;
case ' ':
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
if( '>'!=cBreak && (!bReadListing && !bReadXMP &&
!bReadPRE && !bReadTextArea) )
{
......@@ -787,7 +792,7 @@ int HTMLParser::ScanText( const sal_Unicode cBreak )
{
do {
// All remaining characters make their way into the text.
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
if( MAX_LEN == sTmpBuffer.getLength() )
{
aToken += sTmpBuffer.makeStringAndClear();
......@@ -864,7 +869,7 @@ int HTMLParser::_GetNextRawToken()
}
else if( '!' == nNextCh )
{
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
nNextCh = GetNextChar();
}
......@@ -872,7 +877,7 @@ int HTMLParser::_GetNextRawToken()
while( (HTML_ISALPHA(nNextCh) || '-'==nNextCh) &&
IsParserWorking() && sTmpBuffer.getLength() < MAX_LEN )
{
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
nNextCh = GetNextChar();
}
......@@ -959,7 +964,7 @@ int HTMLParser::_GetNextRawToken()
}
break;
case '-':
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
if( bReadComment )
{
bool bTwoMinus = false;
......@@ -970,7 +975,7 @@ int HTMLParser::_GetNextRawToken()
if( MAX_LEN == sTmpBuffer.getLength() )
aToken += sTmpBuffer.makeStringAndClear();
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
nNextCh = GetNextChar();
}
......@@ -1015,7 +1020,7 @@ int HTMLParser::_GetNextRawToken()
// no break
default:
// all remaining characters are appended to the buffer
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
break;
}
......@@ -1095,7 +1100,7 @@ int HTMLParser::_GetNextToken()
{
OUStringBuffer sTmpBuffer;
do {
sTmpBuffer.append( nNextCh );
sTmpBuffer.appendUtf32( nNextCh );
if( MAX_LEN == sTmpBuffer.getLength() )
aToken += sTmpBuffer.makeStringAndClear();
nNextCh = GetNextChar();
......@@ -1166,10 +1171,10 @@ int HTMLParser::_GetNextToken()
}
bDone = aToken.endsWith( "--" );
if( !bDone )
aToken += OUString(nNextCh);
aToken += OUString(&nNextCh,1);
}
else
aToken += OUString(nNextCh);
aToken += OUString(&nNextCh,1);
if( !bDone )
nNextCh = GetNextChar();
}
......@@ -1261,7 +1266,7 @@ int HTMLParser::_GetNextToken()
bDone = '>'==nNextCh && aToken.endsWith("%");
if( !bDone )
{
aToken += OUString(nNextCh);
aToken += OUString(&nNextCh,1);
nNextCh = GetNextChar();
}
}
......
svtools/source/svrtf/parrtf.cxx
Dosyayı görüntüle @ 4647e778
......@@ -191,7 +191,7 @@ int SvRTFParser::_GetNextToken()
// can be also \{, \}, \'88
for( sal_uInt8 m = 0; m < nUCharOverread; ++m )
{
sal_Unicode cAnsi = nNextCh;
sal_uInt32 cAnsi = nNextCh;
while( 0xD == cAnsi )
cAnsi = GetNextChar();
while( 0xA == cAnsi )
......@@ -382,7 +382,7 @@ void SvRTFParser::ScanText( const sal_Unicode cBreak )
case '}':
case '{':
case '+': // I found in a RTF file
aStrBuffer.append(nNextCh);
aStrBuffer.append(sal_Unicode(nNextCh));
break;
case '~': // nonbreaking space
aStrBuffer.append(static_cast< sal_Unicode >(0xA0));
......@@ -484,7 +484,7 @@ void SvRTFParser::ScanText( const sal_Unicode cBreak )
{
do {
// all other characters end up in the text
aStrBuffer.append(nNextCh);
aStrBuffer.appendUtf32(nNextCh);
if (sal_Unicode(EOF) == (nNextCh = GetNextChar()))
{
......
svtools/source/svrtf/svparser.cxx
Dosyayı görüntüle @ 4647e778
......@@ -22,6 +22,7 @@
#include <tools/debug.hxx>
#include <rtl/textcvt.h>
#include <rtl/tencinfo.h>
#include <rtl/character.hxx>
#include <vector>
......@@ -35,7 +36,7 @@ struct SvParser_Impl
long nTokenValue; // extra value (RTF)
bool bTokenHasValue; // indicates whether nTokenValue is valid
int nToken; // actual Token
sal_Unicode nNextCh; // actual character
sal_uInt32 nNextCh; // actual character
int nSaveToken; // the token from Continue
rtl_TextToUnicodeConverter hConv;
......@@ -148,9 +149,9 @@ void SvParser::RereadLookahead()
nNextCh = GetNextChar();
}
sal_Unicode SvParser::GetNextChar()
sal_uInt32 SvParser::GetNextChar()
{
sal_Unicode c = 0U;
sal_uInt32 c = 0U;
// When reading multiple bytes, we don't have to care about the file
// position when we run into the pending state. The file position is
......@@ -257,7 +258,7 @@ sal_Unicode SvParser::GetNextChar()
)
{
// no convserion shall take place
c = (sal_Unicode)c1;
c = reinterpret_cast<sal_uChar&>( c1 );
nChars = 1;
}
else
......@@ -280,6 +281,7 @@ sal_Unicode SvParser::GetNextChar()
// read enough characters.
if( pImplData->hContext != reinterpret_cast<rtl_TextToUnicodeContext>(1) )
{
sal_Unicode sCh[2];
while( (nInfo&RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOSMALL) != 0 )
{
rInput.ReadChar( c1 );
......@@ -289,7 +291,7 @@ sal_Unicode SvParser::GetNextChar()
nChars = rtl_convertTextToUnicode(
pImplData->hConv, pImplData->hContext,
&c1, 1, &cUC, 1,
&c1, 1, sCh , 2,
RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_ERROR|
RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_ERROR|
RTL_TEXTTOUNICODE_FLAGS_INVALID_ERROR,
......@@ -299,7 +301,11 @@ sal_Unicode SvParser::GetNextChar()
{
if( 1 == nChars && 0 == nInfo )
{
c = cUC;
c = sal_uInt32( sCh[0] );
}
else if( 2 == nChars && 0 == nInfo )
{
c = rtl::combineSurrogates( sCh[0], sCh[1] );
}
else if( 0 != nChars || 0 != nInfo )
{
......@@ -311,7 +317,7 @@ sal_Unicode SvParser::GetNextChar()
"there is a converted character, but an error" );
// There are still errors, but nothing we can
// do
c = (sal_Unicode)'?';
c = (sal_uInt32)'?';
nChars = 1;
}
}
......@@ -356,7 +362,7 @@ sal_Unicode SvParser::GetNextChar()
// There are still errors, so we use the first
// character and restart after that.
c = (sal_Unicode)sBuffer[0];
c = reinterpret_cast<sal_uChar&>( sBuffer[0] );
rInput.SeekRel( -(nLen-1) );
nChars = 1;
}
......@@ -378,7 +384,7 @@ sal_Unicode SvParser::GetNextChar()
"there is no converted character and no error" );
// #73398#: If the character could not be converted,
// because a conversion is not available, do no conversion at all.
c = (sal_Unicode)c1;
c = reinterpret_cast<sal_uChar&>( c1 );
nChars = 1;
}
......@@ -387,6 +393,10 @@ sal_Unicode SvParser::GetNextChar()
}
while( 0 == nChars && !bErr );
}
if ( ! rtl::isValidCodePoint( c ) )
c = (sal_uInt32) '?' ;
if( bErr )
{
if( ERRCODE_IO_PENDING == rInput.GetError() )
......@@ -405,6 +415,7 @@ sal_Unicode SvParser::GetNextChar()
}
else
IncLinePos();
return c;
}
......
sw/qa/extras/htmlexport/data/extb.html 0 → 100644
Dosyayı görüntüle @ 4647e778
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8"/>
</head>
<body>
<p>𤭢</p>
<p>&#x24b62;</p>
</body>
</html>
sw/qa/extras/htmlexport/htmlexport.cxx
Dosyayı görüntüle @ 4647e778
......@@ -272,6 +272,19 @@ DECLARE_HTMLEXPORT_TEST(testTdf83890, "tdf83890.odt")
assertXPath(pDoc, "/html/body/ol[2]/ol", "start", "2");
}
DECLARE_HTMLEXPORT_TEST(testExtbChars, "extb.html")
{
sal_uInt32 nCh = 0x24b62;
OUString aExpected( &nCh, 1);
// Assert that UTF8 encoded non-BMP Unicode character is correct
uno::Reference<text::XTextRange> xTextRange1 = getRun(getParagraph(1), 1);
CPPUNIT_ASSERT_EQUAL(aExpected, xTextRange1->getString());
// Assert that non-BMP Unicode in character entity format is correct
uno::Reference<text::XTextRange> xTextRange2 = getRun(getParagraph(2), 1);
CPPUNIT_ASSERT_EQUAL(aExpected, xTextRange2->getString());
}
CPPUNIT_PLUGIN_IMPLEMENT();
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment