[3.4] bpo-29591, bpo-30694: Upgrade Modules/expat to libexpat 2.2.1 (#2164) (#2203)

* bpo-29591: Upgrade Modules/expat to libexpat 2.2 (#2164)

* bpo-29591: Upgrade Modules/expat to libexpat 2.2

* bpo-29591: Restore Python changes on expat

* bpo-29591: Remove expat config of unsupported platforms

Remove the configuration (Modules/expat/*config.h) of unsupported
platforms:

* Amiga
* MacOS Classic on PPC32
* Open Watcom

* bpo-29591: Remove useless XML_HAS_SET_HASH_SALT

The XML_HAS_SET_HASH_SALT define of Modules/expat/expat.h became
useless since our local expat copy was upgrade to expat 2.1 (it's now
expat 2.2.0).

(cherry picked from commit 23ec4b57)

* bpo-30694: Upgrade Modules/expat/ to libexpat 2.2.1 (#2300)

New file: Modules/expat/siphash.h.
(cherry picked from commit 5ff71323)

* bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348)

bpo-30726, bpo-29591: libexpat 2.2.1 of Modules/expat/ now uses
a winconfig.h configuration file which already defines:

* XML_NS
* XML_DTD
* BYTEORDER=1234
* XML_CONTEXT_BYTES=1024
* HAVE_MEMMOVE

Remove these defines from PCbuild/_elementtree.vcxproj to prevent
compiler warnings.
Co-Authored-By: Jeremy Kloth <jeremy.kloth@gmail.com>
(cherry picked from commit c8fb58bd)

* bpo-30726: Fix elementtree warnings on Windows due to expat upgrade (#2319)

* bpo-30726: Fix elementtree warnings on Windows

Caused by usage of `getenv` which should be safe. And a few integer
truncations which should also be ok.

* bpo-30726: Don't ignore libexpat warnings which haypo intends to fix upstream

(cherry picked from commit 87c65550)

Misc/NEWS.d/next/Security/2017-07-11-22-25-24.bpo-30694.oOf3Er.rst

+Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security
+vulnerabilities including: CVE-2017-9233 (External entity infinite loop
+DoS), CVE-2016-9063 (Integer overflow, re-fix), CVE-2016-0718 (Fix
+regression bugs from 2.2.0's fix to CVE-2016-0718) and CVE-2012-0876
+(Counter hash flooding with SipHash).  Note: the CVE-2016-5300 (Use os-
+specific entropy sources like getrandom) doesn't impact Python, since Python
+already gets entropy from the OS to set the expat secret using
+``XML_SetHashSalt()``.

Misc/NEWS.d/next/Security/2017-07-11-22-26-48.bpo-29591.cOeMX-.rst

+Update expat copy from 2.1.1 to 2.2.0 to get fixes of CVE-2016-0718 and
+CVE-2016-4472. See https://sourceforge.net/p/expat/bugs/537/ for more
+information.

Modules/expat/COPYING

-Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd
-                               and Clark Cooper
-Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers.
+Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper
+Copyright (c) 2001-2017 Expat maintainers
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

Modules/expat/amigaconfig.h

-#ifndef AMIGACONFIG_H
-#define AMIGACONFIG_H
-
-/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */
-#define BYTEORDER 4321
-
-/* Define to 1 if you have the `bcopy' function. */
-#define HAVE_BCOPY 1
-
-/* Define to 1 if you have the <check.h> header file. */
-#undef HAVE_CHECK_H
-
-/* Define to 1 if you have the `memmove' function. */
-#define HAVE_MEMMOVE 1
-
-/* Define to 1 if you have the <unistd.h> header file. */
-#define HAVE_UNISTD_H 1
-
-/* whether byteorder is bigendian */
-#define WORDS_BIGENDIAN
-
-/* Define to specify how much context to retain around the current parse
-   point. */
-#define XML_CONTEXT_BYTES 1024
-
-/* Define to make parameter entity parsing functionality available. */
-#define XML_DTD
-
-/* Define to make XML Namespaces functionality available. */
-#define XML_NS
-
-#endif  /* AMIGACONFIG_H */

Modules/expat/expat.h

@@ -95,7 +95,9 @@ enum XML_Error {
   /* Added in 2.0. */
   XML_ERROR_RESERVED_PREFIX_XML,
   XML_ERROR_RESERVED_PREFIX_XMLNS,
-  XML_ERROR_RESERVED_NAMESPACE_URI
+  XML_ERROR_RESERVED_NAMESPACE_URI,
+  /* Added in 2.2.1. */
+  XML_ERROR_INVALID_ARGUMENT
 };
 
 enum XML_Content_Type {
@@ -342,7 +344,7 @@ XML_SetEntityDeclHandler(XML_Parser parser,
                          XML_EntityDeclHandler handler);
 
 /* OBSOLETE -- OBSOLETE -- OBSOLETE
-   This handler has been superceded by the EntityDeclHandler above.
+   This handler has been superseded by the EntityDeclHandler above.
    It is provided here for backward compatibility.
 
    This is called for a declaration of an unparsed (NDATA) entity.
@@ -706,6 +708,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
      be called, despite an external subset being parsed.
    Note: If XML_DTD is not defined when Expat is compiled, returns
      XML_ERROR_FEATURE_REQUIRES_XML_DTD.
+   Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT.
 */
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
@@ -729,15 +732,16 @@ XML_GetBase(XML_Parser parser);
    to the XML_StartElementHandler that were specified in the start-tag
    rather than defaulted. Each attribute/value pair counts as 2; thus
    this correspondds to an index into the atts array passed to the
-   XML_StartElementHandler.
+   XML_StartElementHandler.  Returns -1 if parser == NULL.
 */
 XMLPARSEAPI(int)
 XML_GetSpecifiedAttributeCount(XML_Parser parser);
 
 /* Returns the index of the ID attribute passed in the last call to
-   XML_StartElementHandler, or -1 if there is no ID attribute.  Each
-   attribute/value pair counts as 2; thus this correspondds to an
-   index into the atts array passed to the XML_StartElementHandler.
+   XML_StartElementHandler, or -1 if there is no ID attribute or
+   parser == NULL.  Each attribute/value pair counts as 2; thus this
+   correspondds to an index into the atts array passed to the
+   XML_StartElementHandler.
 */
 XMLPARSEAPI(int)
 XML_GetIdAttributeIndex(XML_Parser parser);
@@ -901,6 +905,7 @@ enum XML_ParamEntityParsing {
    entities is requested; otherwise it will return non-zero.
    Note: If XML_SetParamEntityParsing is called after XML_Parse or
       XML_ParseBuffer, then it has no effect and will always return 0.
+   Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
 XML_SetParamEntityParsing(XML_Parser parser,
@@ -910,13 +915,12 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Helps in preventing DoS attacks based on predicting hash
    function behavior. This must be called before parsing is started.
    Returns 1 if successful, 0 when called after parsing has started.
+   Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
 XML_SetHashSalt(XML_Parser parser,
                 unsigned long hash_salt);
 
-#define XML_HAS_SET_HASH_SALT  /* Python Only: Defined for pyexpat.c. */
-
 /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then
    XML_GetErrorCode returns information about the error.
 */
@@ -938,6 +942,10 @@ XML_GetErrorCode(XML_Parser parser);
    the location is the location of the character at which the error
    was detected; otherwise the location is the location of the last
    parse event, as described above.
+
+   Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber
+   return 0 to indicate an error.
+   Note: XML_GetCurrentByteIndex returns -1 to indicate an error.
 */
 XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser);
 XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser);
@@ -975,9 +983,12 @@ XML_FreeContentModel(XML_Parser parser, XML_Content *model);
 
 /* Exposing the memory handling functions used in Expat */
 XMLPARSEAPI(void *)
+XML_ATTR_MALLOC
+XML_ATTR_ALLOC_SIZE(2)
 XML_MemMalloc(XML_Parser parser, size_t size);
 
 XMLPARSEAPI(void *)
+XML_ATTR_ALLOC_SIZE(3)
 XML_MemRealloc(XML_Parser parser, void *ptr, size_t size);
 
 XMLPARSEAPI(void)
@@ -1033,13 +1044,11 @@ XMLPARSEAPI(const XML_Feature *)
 XML_GetFeatureList(void);
 
 
-/* Expat follows the GNU/Linux convention of odd number minor version for
-   beta/development releases and even number minor version for stable
-   releases. Micro is bumped with each release, and set to 0 with each
-   change to major or minor version.
+/* Expat follows the semantic versioning convention.
+   See http://semver.org.
 */
 #define XML_MAJOR_VERSION 2
-#define XML_MINOR_VERSION 1
+#define XML_MINOR_VERSION 2
 #define XML_MICRO_VERSION 1
 
 #ifdef __cplusplus

Modules/expat/expat_external.h

@@ -69,12 +69,26 @@
 #endif
 #endif  /* not defined XML_STATIC */
 
+#if !defined(XMLIMPORT) && defined(__GNUC__) && (__GNUC__ >= 4)
+#define XMLIMPORT __attribute__ ((visibility ("default")))
+#endif
 
 /* If we didn't define it above, define it away: */
 #ifndef XMLIMPORT
 #define XMLIMPORT
 #endif
 
+#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96))
+#define XML_ATTR_MALLOC __attribute__((__malloc__))
+#else
+#define XML_ATTR_MALLOC
+#endif
+
+#if defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#define XML_ATTR_ALLOC_SIZE(x)  __attribute__((__alloc_size__(x)))
+#else
+#define XML_ATTR_ALLOC_SIZE(x)
+#endif
 
 #define XMLPARSEAPI(type) XMLIMPORT type XMLCALL
 
@@ -83,7 +97,10 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-#define XML_UNICODE
+# define XML_UNICODE
+# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+# endif
 #endif
 
 #ifdef XML_UNICODE     /* Information is UTF-16 encoded. */

Modules/expat/internal.h

@@ -71,3 +71,25 @@
 #define inline
 #endif
 #endif
+
+#ifndef UNUSED_P
+# ifdef __GNUC__
+#  define UNUSED_P(p) UNUSED_ ## p __attribute__((__unused__))
+# else
+#  define UNUSED_P(p) UNUSED_ ## p
+# endif
+#endif
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+void
+align_limit_to_full_utf8_characters(const char * from, const char ** fromLimRef);
+
+
+#ifdef __cplusplus
+}
+#endif

Modules/expat/macconfig.h

-/*================================================================
-** Copyright 2000, Clark Cooper
-** All rights reserved.
-**
-** This is free software. You are permitted to copy, distribute, or modify
-** it under the terms of the MIT/X license (contained in the COPYING file
-** with this distribution.)
-**
-*/
-
-#ifndef MACCONFIG_H
-#define MACCONFIG_H
-
-
-/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */
-#define BYTEORDER  4321
-
-/* Define to 1 if you have the `bcopy' function. */
-#undef HAVE_BCOPY
-
-/* Define to 1 if you have the `memmove' function. */
-#define HAVE_MEMMOVE
-
-/* Define to 1 if you have a working `mmap' system call. */
-#undef HAVE_MMAP
-
-/* Define to 1 if you have the <unistd.h> header file. */
-#undef HAVE_UNISTD_H
-
-/* whether byteorder is bigendian */
-#define WORDS_BIGENDIAN
-
-/* Define to specify how much context to retain around the current parse
-   point. */
-#undef XML_CONTEXT_BYTES
-
-/* Define to make parameter entity parsing functionality available. */
-#define XML_DTD
-
-/* Define to make XML Namespaces functionality available. */
-#define XML_NS
-
-/* Define to empty if `const' does not conform to ANSI C. */
-#undef const
-
-/* Define to `long' if <sys/types.h> does not define. */
-#define off_t  long
-
-/* Define to `unsigned' if <sys/types.h> does not define. */
-#undef size_t
-
-
-#endif /* ifndef MACCONFIG_H */

Modules/expat/watcomconfig.h

-/* expat_config.h for use with Open Watcom 1.5 and above.  */
-
-#ifndef WATCOMCONFIG_H
-#define WATCOMCONFIG_H
-
-#ifdef __NT__
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#undef WIN32_LEAN_AND_MEAN
-#endif
-
-/* 1234 = LIL_ENDIAN, 4321 = BIGENDIAN */
-#define BYTEORDER 1234
-
-/* Define to 1 if you have the `memmove' function. */
-#define HAVE_MEMMOVE 1
-
-/* Define to 1 if you have the <unistd.h> header file. */
-#define HAVE_UNISTD_H 1
-
-/* Define to the address where bug reports for this package should be sent. */
-#define PACKAGE_BUGREPORT "expat-bugs@mail.libexpat.org"
-
-/* Define to the full name of this package. */
-#define PACKAGE_NAME "expat"
-
-/* Define to the full name and version of this package. */
-#define PACKAGE_STRING "expat 2.0.0"
-
-/* Define to the one symbol short name of this package. */
-#undef PACKAGE_TARNAME
-
-/* Define to the version of this package. */
-#define PACKAGE_VERSION "2.0.0"
-
-/* Define to specify how much context to retain around the current parse
-   point. */
-#define XML_CONTEXT_BYTES 1024
-
-/* Define to make parameter entity parsing functionality available. */
-#define XML_DTD 1
-
-/* Define to make XML Namespaces functionality available. */
-#define XML_NS 1
-
-#endif
-

Modules/expat/winconfig.h

@@ -17,6 +17,12 @@
 #include <memory.h>
 #include <string.h>
 
+
+#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
+# include <expat_config.h>
+#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
+
+
 #define XML_NS 1
 #define XML_DTD 1
 #define XML_CONTEXT_BYTES 1024
@@ -27,4 +33,8 @@
 /* Windows has memmove() available. */
 #define HAVE_MEMMOVE
 
+
+#endif /* !defined(HAVE_EXPAT_CONFIG_H) */
+
+
 #endif /* ndef WINCONFIG_H */

Modules/expat/xmltok.h

@@ -130,6 +130,12 @@ typedef int (PTRCALL *SCANNER)(const ENCODING *,
                                const char *,
                                const char **);
 
+enum XML_Convert_Result {
+  XML_CONVERT_COMPLETED = 0,
+  XML_CONVERT_INPUT_INCOMPLETE = 1,
+  XML_CONVERT_OUTPUT_EXHAUSTED = 2  /* and therefore potentially input remaining as well */
+};
+
 struct encoding {
   SCANNER scanners[XML_N_STATES];
   SCANNER literalScanners[XML_N_LITERAL_TYPES];
@@ -158,12 +164,12 @@ struct encoding {
                             const char *ptr,
                             const char *end,
                             const char **badPtr);
-  void (PTRCALL *utf8Convert)(const ENCODING *enc,
+  enum XML_Convert_Result (PTRCALL *utf8Convert)(const ENCODING *enc,
                               const char **fromP,
                               const char *fromLim,
                               char **toP,
                               const char *toLim);
-  void (PTRCALL *utf16Convert)(const ENCODING *enc,
+  enum XML_Convert_Result (PTRCALL *utf16Convert)(const ENCODING *enc,
                                const char **fromP,
                                const char *fromLim,
                                unsigned short **toP,

Modules/pyexpat.c

@@ -1107,10 +1107,8 @@ newxmlparseobject(char *encoding, char *namespace_separator, PyObject *intern)
         Py_DECREF(self);
         return NULL;
     }
-#if ((XML_MAJOR_VERSION >= 2) && (XML_MINOR_VERSION >= 1)) || defined(XML_HAS_SET_HASH_SALT)
-    /* This feature was added upstream in libexpat 2.1.0.  Our expat copy
-     * has a backport of this feature where we also define XML_HAS_SET_HASH_SALT
-     * to indicate that we can still use it. */
+#if XML_COMBINED_VERSION >= 20100
+    /* This feature was added upstream in libexpat 2.1.0. */
     XML_SetHashSalt(self->itself,
                     (unsigned long)_Py_HashSecret.expat.hashsalt);
 #endif

PCbuild/_elementtree.vcxproj

@@ -164,7 +164,7 @@
     </Midl>
     <ClCompile>
       <AdditionalIncludeDirectories>..\Modules\expat;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;USE_PYEXPAT_CAPI;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
       <BaseAddress>0x1D100000</BaseAddress>
@@ -267,4 +267,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
\ No newline at end of file
+</Project>