Skip to content

C
cpython
Kaydet (Commit) d9491269 authored tarafından Victor Stinner's avatar Victor Stinner
Dosyalara gözat
Seçenekler

Issue #17693: CJK encoders now use the new Unicode API (PEP 393)

üst 71557596
Expand all Show whitespace changes
Inline Side-by-side
Showing with 247 additions and 259 deletions
Modules/cjkcodecs/_codecs_cn.c
Dosyayı görüntüle @ d9491269
......@@ -42,16 +42,18 @@
ENCODER(gb2312)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
TRYMAP_ENC(gbcommon, code, c);
......@@ -60,9 +62,9 @@ ENCODER(gb2312)
if (code & 0x8000) /* MSB set: GBK */
return 1;
OUT1((code >> 8) | 0x80)
OUT2((code & 0xFF) | 0x80)
NEXT(1, 2)
OUTBYTE1((code >> 8) | 0x80)
OUTBYTE2((code & 0xFF) | 0x80)
NEXT(1, 2);
}
return 0;
......@@ -80,7 +82,7 @@ DECODER(gb2312)
}
REQUIRE_INBUF(2)
TRYMAP_DEC(gb2312, writer, c ^ 0x80, IN2 ^ 0x80) {
TRYMAP_DEC(gb2312, writer, c ^ 0x80, INBYTE2 ^ 0x80) {
NEXT_IN(2);
}
else return 1;
......@@ -96,28 +98,30 @@ DECODER(gb2312)
ENCODER(gbk)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
GBK_ENCODE(c, code)
else return 1;
OUT1((code >> 8) | 0x80)
OUTBYTE1((code >> 8) | 0x80)
if (code & 0x8000)
OUT2((code & 0xFF)) /* MSB set: GBK */
OUTBYTE2((code & 0xFF)) /* MSB set: GBK */
else
OUT2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */
NEXT(1, 2)
OUTBYTE2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */
NEXT(1, 2);
}
return 0;
......@@ -126,7 +130,7 @@ ENCODER(gbk)
DECODER(gbk)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
if (c < 0x80) {
OUTCHAR(c);
......@@ -136,7 +140,7 @@ DECODER(gbk)
REQUIRE_INBUF(2)
GBK_DECODE(c, IN2, writer)
GBK_DECODE(c, INBYTE2, writer)
else return 1;
NEXT_IN(2);
......@@ -152,41 +156,31 @@ DECODER(gbk)
ENCODER(gb18030)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1(c)
NEXT(1, 1)
WRITEBYTE1(c)
NEXT(1, 1);
continue;
}
DECODE_SURROGATE(c)
if (c > 0x10FFFF)
#if Py_UNICODE_SIZE == 2
return 2; /* surrogates pair */
#else
return 1;
#endif
else if (c >= 0x10000) {
if (c >= 0x10000) {
Py_UCS4 tc = c - 0x10000;
assert (c <= 0x10FFFF);
REQUIRE_OUTBUF(4)
OUT4((unsigned char)(tc % 10) + 0x30)
OUTBYTE4((unsigned char)(tc % 10) + 0x30)
tc /= 10;
OUT3((unsigned char)(tc % 126) + 0x81)
OUTBYTE3((unsigned char)(tc % 126) + 0x81)
tc /= 126;
OUT2((unsigned char)(tc % 10) + 0x30)
OUTBYTE2((unsigned char)(tc % 10) + 0x30)
tc /= 10;
OUT1((unsigned char)(tc + 0x90))
OUTBYTE1((unsigned char)(tc + 0x90))
#if Py_UNICODE_SIZE == 2
NEXT(2, 4) /* surrogates pair */
#else
NEXT(1, 4)
#endif
NEXT(1, 4);
continue;
}
......@@ -209,15 +203,15 @@ ENCODER(gb18030)
tc = c - utrrange->first +
utrrange->base;
OUT4((unsigned char)(tc % 10) + 0x30)
OUTBYTE4((unsigned char)(tc % 10) + 0x30)
tc /= 10;
OUT3((unsigned char)(tc % 126) + 0x81)
OUTBYTE3((unsigned char)(tc % 126) + 0x81)
tc /= 126;
OUT2((unsigned char)(tc % 10) + 0x30)
OUTBYTE2((unsigned char)(tc % 10) + 0x30)
tc /= 10;
OUT1((unsigned char)tc + 0x81)
OUTBYTE1((unsigned char)tc + 0x81)
NEXT(1, 4)
NEXT(1, 4);
break;
}
......@@ -226,13 +220,13 @@ ENCODER(gb18030)
continue;
}
OUT1((code >> 8) | 0x80)
OUTBYTE1((code >> 8) | 0x80)
if (code & 0x8000)
OUT2((code & 0xFF)) /* MSB set: GBK or GB18030ext */
OUTBYTE2((code & 0xFF)) /* MSB set: GBK or GB18030ext */
else
OUT2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */
OUTBYTE2((code & 0xFF) | 0x80) /* MSB unset: GB2312 */
NEXT(1, 2)
NEXT(1, 2);
}
return 0;
......@@ -241,7 +235,7 @@ ENCODER(gb18030)
DECODER(gb18030)
{
while (inleft > 0) {
unsigned char c = IN1, c2;
unsigned char c = INBYTE1, c2;
if (c < 0x80) {
OUTCHAR(c);
......@@ -251,15 +245,15 @@ DECODER(gb18030)
REQUIRE_INBUF(2)
c2 = IN2;
c2 = INBYTE2;
if (c2 >= 0x30 && c2 <= 0x39) { /* 4 bytes seq */
const struct _gb18030_to_unibmp_ranges *utr;
unsigned char c3, c4;
Py_UCS4 lseq;
REQUIRE_INBUF(4)
c3 = IN3;
c4 = IN4;
c3 = INBYTE3;
c4 = INBYTE4;
if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39)
return 1;
c -= 0x81; c2 -= 0x30;
......@@ -313,33 +307,34 @@ ENCODER_INIT(hz)
ENCODER_RESET(hz)
{
if (state->i != 0) {
WRITE2('~', '}')
WRITEBYTE2('~', '}')
state->i = 0;
NEXT_OUT(2)
NEXT_OUT(2);
}
return 0;
}
ENCODER(hz)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
if (state->i == 0) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
}
else {
WRITE3('~', '}', (unsigned char)c)
NEXT(1, 3)
WRITEBYTE3('~', '}', (unsigned char)c)
NEXT(1, 3);
state->i = 0;
}
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
TRYMAP_ENC(gbcommon, code, c);
else return 1;
......@@ -348,13 +343,13 @@ ENCODER(hz)
return 1;
if (state->i == 0) {
WRITE4('~', '{', code >> 8, code & 0xff)
NEXT(1, 4)
WRITEBYTE4('~', '{', code >> 8, code & 0xff)
NEXT(1, 4);
state->i = 1;
}
else {
WRITE2(code >> 8, code & 0xff)
NEXT(1, 2)
WRITEBYTE2(code >> 8, code & 0xff)
NEXT(1, 2);
}
}
......@@ -376,10 +371,10 @@ DECODER_RESET(hz)
DECODER(hz)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
if (c == '~') {
unsigned char c2 = IN2;
unsigned char c2 = INBYTE2;
REQUIRE_INBUF(2)
if (c2 == '~') {
......@@ -408,7 +403,7 @@ DECODER(hz)
}
else { /* GB mode */
REQUIRE_INBUF(2)
TRYMAP_DEC(gb2312, writer, c, IN2) {
TRYMAP_DEC(gb2312, writer, c, INBYTE2) {
NEXT_IN(2);
}
else
......
Modules/cjkcodecs/_codecs_hk.c
Dosyayı görüntüle @ d9491269
......@@ -38,35 +38,39 @@ static const DBCHAR big5hkscs_pairenc_table[4] = {0x8862, 0x8864, 0x88a3, 0x88a5
ENCODER(big5hkscs)
{
while (inleft > 0) {
Py_UCS4 c = **inbuf;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
Py_ssize_t insize;
if (c < 0x80) {
REQUIRE_OUTBUF(1)
**outbuf = (unsigned char)c;
NEXT(1, 1)
NEXT(1, 1);
continue;
}
DECODE_SURROGATE(c)
insize = GET_INSIZE(c);
insize = 1;
REQUIRE_OUTBUF(2)
if (c < 0x10000) {
TRYMAP_ENC(big5hkscs_bmp, code, c) {
if (code == MULTIC) {
if (inleft >= 2 &&
Py_UCS4 c2;
if (inlen - *inpos >= 2)
c2 = INCHAR2;
else
c2 = 0;
if (inlen - *inpos >= 2 &&
((c & 0xffdf) == 0x00ca) &&
(((*inbuf)[1] & 0xfff7) == 0x0304)) {
((c2 & 0xfff7) == 0x0304)) {
code = big5hkscs_pairenc_table[
((c >> 4) |
((*inbuf)[1] >> 3)) & 3];
(c2 >> 3)) & 3];
insize = 2;
}
else if (inleft < 2 &&
else if (inlen - *inpos < 2 &&
!(flags & MBENC_FLUSH))
return MBERR_TOOFEW;
else {
......@@ -89,9 +93,9 @@ ENCODER(big5hkscs)
else
return insize;
OUT1(code >> 8)
OUT2(code & 0xFF)
NEXT(insize, 2)
OUTBYTE1(code >> 8)
OUTBYTE2(code & 0xFF)
NEXT(insize, 2);
}
return 0;
......@@ -102,7 +106,7 @@ ENCODER(big5hkscs)
DECODER(big5hkscs)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
Py_UCS4 decoded;
if (c < 0x80) {
......@@ -113,20 +117,20 @@ DECODER(big5hkscs)
REQUIRE_INBUF(2)
if (0xc6 > c || c > 0xc8 || (c < 0xc7 && IN2 < 0xa1)) {
TRYMAP_DEC(big5, writer, c, IN2) {
if (0xc6 > c || c > 0xc8 || (c < 0xc7 && INBYTE2 < 0xa1)) {
TRYMAP_DEC(big5, writer, c, INBYTE2) {
NEXT_IN(2);
continue;
}
}
TRYMAP_DEC_CHAR(big5hkscs, decoded, c, IN2)
TRYMAP_DEC_CHAR(big5hkscs, decoded, c, INBYTE2)
{
int s = BH2S(c, IN2);
int s = BH2S(c, INBYTE2);
const unsigned char *hintbase;
assert(0x87 <= c && c <= 0xfe);
assert(0x40 <= IN2 && IN2 <= 0xfe);
assert(0x40 <= INBYTE2 && INBYTE2 <= 0xfe);
if (BH2S(0x87, 0x40) <= s && s <= BH2S(0xa0, 0xfe)) {
hintbase = big5hkscs_phint_0;
......@@ -154,7 +158,7 @@ DECODER(big5hkscs)
continue;
}
switch ((c << 8) | IN2) {
switch ((c << 8) | INBYTE2) {
case 0x8862: OUTCHAR2(0x00ca, 0x0304); break;
case 0x8864: OUTCHAR2(0x00ca, 0x030c); break;
case 0x88a3: OUTCHAR2(0x00ea, 0x0304); break;
......
Modules/cjkcodecs/_codecs_iso2022.c
Dosyayı görüntüle @ d9491269
......@@ -141,13 +141,13 @@ ENCODER_INIT(iso2022)
ENCODER_RESET(iso2022)
{
if (STATE_GETFLAG(F_SHIFTED)) {
WRITE1(SI)
NEXT_OUT(1)
WRITEBYTE1(SI)
NEXT_OUT(1);
STATE_CLEARFLAG(F_SHIFTED)
}
if (STATE_G0 != CHARSET_ASCII) {
WRITE3(ESC, '(', 'B')
NEXT_OUT(3)
WRITEBYTE3(ESC, '(', 'B')
NEXT_OUT(3);
STATE_SETG0(CHARSET_ASCII)
}
return 0;
......@@ -155,30 +155,29 @@ ENCODER_RESET(iso2022)
ENCODER(iso2022)
{
while (inleft > 0) {
while (*inpos < inlen) {
const struct iso2022_designation *dsg;
DBCHAR encoded;
Py_UCS4 c = **inbuf;
Py_UCS4 c = INCHAR1;
Py_ssize_t insize;
if (c < 0x80) {
if (STATE_G0 != CHARSET_ASCII) {
WRITE3(ESC, '(', 'B')
WRITEBYTE3(ESC, '(', 'B')
STATE_SETG0(CHARSET_ASCII)
NEXT_OUT(3)
NEXT_OUT(3);
}
if (STATE_GETFLAG(F_SHIFTED)) {
WRITE1(SI)
WRITEBYTE1(SI)
STATE_CLEARFLAG(F_SHIFTED)
NEXT_OUT(1)
NEXT_OUT(1);
}
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
DECODE_SURROGATE(c)
insize = GET_INSIZE(c);
insize = 1;
encoded = MAP_UNMAPPABLE;
for (dsg = CONFIG_DESIGNATIONS; dsg->mark; dsg++) {
......@@ -187,24 +186,14 @@ ENCODER(iso2022)
if (encoded == MAP_MULTIPLE_AVAIL) {
/* this implementation won't work for pair
* of non-bmp characters. */
if (inleft < 2) {
if (inlen - *inpos < 2) {
if (!(flags & MBENC_FLUSH))
return MBERR_TOOFEW;
length = -1;
}
else
length = 2;
#if Py_UNICODE_SIZE == 2
if (length == 2) {
Py_UCS4 u4in[2];
u4in[0] = (Py_UCS4)IN1;
u4in[1] = (Py_UCS4)IN2;
encoded = dsg->encoder(u4in, &length);
} else
encoded = dsg->encoder(&c, &length);
#else
encoded = dsg->encoder(&c, &length);
#endif
if (encoded != MAP_UNMAPPABLE) {
insize = length;
break;
......@@ -221,47 +210,47 @@ ENCODER(iso2022)
switch (dsg->plane) {
case 0: /* G0 */
if (STATE_GETFLAG(F_SHIFTED)) {
WRITE1(SI)
WRITEBYTE1(SI)
STATE_CLEARFLAG(F_SHIFTED)
NEXT_OUT(1)
NEXT_OUT(1);
}
if (STATE_G0 != dsg->mark) {
if (dsg->width == 1) {
WRITE3(ESC, '(', ESCMARK(dsg->mark))
WRITEBYTE3(ESC, '(', ESCMARK(dsg->mark))
STATE_SETG0(dsg->mark)
NEXT_OUT(3)
NEXT_OUT(3);
}
else if (dsg->mark == CHARSET_JISX0208) {
WRITE3(ESC, '$', ESCMARK(dsg->mark))
WRITEBYTE3(ESC, '$', ESCMARK(dsg->mark))
STATE_SETG0(dsg->mark)
NEXT_OUT(3)
NEXT_OUT(3);
}
else {
WRITE4(ESC, '$', '(',
WRITEBYTE4(ESC, '$', '(',
ESCMARK(dsg->mark))
STATE_SETG0(dsg->mark)
NEXT_OUT(4)
NEXT_OUT(4);
}
}
break;
case 1: /* G1 */
if (STATE_G1 != dsg->mark) {
if (dsg->width == 1) {
WRITE3(ESC, ')', ESCMARK(dsg->mark))
WRITEBYTE3(ESC, ')', ESCMARK(dsg->mark))
STATE_SETG1(dsg->mark)
NEXT_OUT(3)
NEXT_OUT(3);
}
else {
WRITE4(ESC, '$', ')',
WRITEBYTE4(ESC, '$', ')',
ESCMARK(dsg->mark))
STATE_SETG1(dsg->mark)
NEXT_OUT(4)
NEXT_OUT(4);
}
}
if (!STATE_GETFLAG(F_SHIFTED)) {
WRITE1(SO)
WRITEBYTE1(SO)
STATE_SETFLAG(F_SHIFTED)
NEXT_OUT(1)
NEXT_OUT(1);
}
break;
default: /* G2 and G3 is not supported: no encoding in
......@@ -270,14 +259,14 @@ ENCODER(iso2022)
}
if (dsg->width == 1) {
WRITE1((unsigned char)encoded)
NEXT_OUT(1)
WRITEBYTE1((unsigned char)encoded)
NEXT_OUT(1);
}
else {
WRITE2(encoded >> 8, encoded & 0xff)
NEXT_OUT(2)
WRITEBYTE2(encoded >> 8, encoded & 0xff)
NEXT_OUT(2);
}
NEXT_IN(insize);
NEXT_INCHAR(insize);
}
return 0;
......@@ -323,26 +312,26 @@ iso2022processesc(const void *config, MultibyteCodec_State *state,
switch (esclen) {
case 3:
if (IN2 == '$') {
charset = IN3 | CHARSET_DBCS;
if (INBYTE2 == '$') {
charset = INBYTE3 | CHARSET_DBCS;
designation = 0;
}
else {
charset = IN3;
if (IN2 == '(') designation = 0;
else if (IN2 == ')') designation = 1;
else if (CONFIG_ISSET(USE_G2) && IN2 == '.')
charset = INBYTE3;
if (INBYTE2 == '(') designation = 0;
else if (INBYTE2 == ')') designation = 1;
else if (CONFIG_ISSET(USE_G2) && INBYTE2 == '.')
designation = 2;
else return 3;
}
break;
case 4:
if (IN2 != '$')
if (INBYTE2 != '$')
return 4;
charset = IN4 | CHARSET_DBCS;
if (IN3 == '(') designation = 0;
else if (IN3 == ')') designation = 1;
charset = INBYTE4 | CHARSET_DBCS;
if (INBYTE3 == '(') designation = 0;
else if (INBYTE3 == ')') designation = 1;
else return 4;
break;
case 6: /* designation with prefix */
......@@ -395,18 +384,18 @@ iso2022processg2(const void *config, MultibyteCodec_State *state,
/* not written to use encoder, decoder functions because only few
* encodings use G2 designations in CJKCodecs */
if (STATE_G2 == CHARSET_ISO8859_1) {
if (IN3 < 0x80)
OUTCHAR(IN3 + 0x80);
if (INBYTE3 < 0x80)
OUTCHAR(INBYTE3 + 0x80);
else
return 3;
}
else if (STATE_G2 == CHARSET_ISO8859_7) {
ISO8859_7_DECODE(IN3 ^ 0x80, writer)
ISO8859_7_DECODE(INBYTE3 ^ 0x80, writer)
else return 3;
}
else if (STATE_G2 == CHARSET_ASCII) {
if (IN3 & 0x80) return 3;
else OUTCHAR(IN3);
if (INBYTE3 & 0x80) return 3;
else OUTCHAR(INBYTE3);
}
else
return MBERR_INTERNAL;
......@@ -421,7 +410,7 @@ DECODER(iso2022)
const struct iso2022_designation *dsgcache = NULL;
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
Py_ssize_t err;
if (STATE_GETFLAG(F_ESCTHROUGHOUT)) {
......@@ -438,13 +427,13 @@ DECODER(iso2022)
switch (c) {
case ESC:
REQUIRE_INBUF(2)
if (IS_ISO2022ESC(IN2)) {
if (IS_ISO2022ESC(INBYTE2)) {
err = iso2022processesc(config, state,
inbuf, &inleft);
if (err != 0)
return err;
}
else if (CONFIG_ISSET(USE_G2) && IN2 == 'N') {/* SS2 */
else if (CONFIG_ISSET(USE_G2) && INBYTE2 == 'N') {/* SS2 */
REQUIRE_INBUF(3)
err = iso2022processg2(config, state,
inbuf, &inleft, writer);
......
Modules/cjkcodecs/_codecs_jp.c
Dosyayı görüntüle @ d9491269
This diff is collapsed.
Modules/cjkcodecs/_codecs_kr.c
Dosyayı görüntüle @ d9491269
......@@ -33,16 +33,18 @@ static const unsigned char u2cgk_jongseong[28] = {
ENCODER(euc_kr)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
TRYMAP_ENC(cp949, code, c);
......@@ -50,9 +52,9 @@ ENCODER(euc_kr)
if ((code & 0x8000) == 0) {
/* KS X 1001 coded character */
OUT1((code >> 8) | 0x80)
OUT2((code & 0xFF) | 0x80)
NEXT(1, 2)
OUTBYTE1((code >> 8) | 0x80)
OUTBYTE2((code & 0xFF) | 0x80)
NEXT(1, 2);
}
else { /* Mapping is found in CP949 extension,
* but we encode it in KS X 1001:1998 Annex 3,
......@@ -61,23 +63,23 @@ ENCODER(euc_kr)
REQUIRE_OUTBUF(8)
/* syllable composition precedence */
OUT1(EUCKR_JAMO_FIRSTBYTE)
OUT2(EUCKR_JAMO_FILLER)
OUTBYTE1(EUCKR_JAMO_FIRSTBYTE)
OUTBYTE2(EUCKR_JAMO_FILLER)
/* All codepoints in CP949 extension are in unicode
* Hangul Syllable area. */
assert(0xac00 <= c && c <= 0xd7a3);
c -= 0xac00;
OUT3(EUCKR_JAMO_FIRSTBYTE)
OUT4(u2cgk_choseong[c / 588])
NEXT_OUT(4)
OUTBYTE3(EUCKR_JAMO_FIRSTBYTE)
OUTBYTE4(u2cgk_choseong[c / 588])
NEXT_OUT(4);
OUT1(EUCKR_JAMO_FIRSTBYTE)
OUT2(u2cgk_jungseong[(c / 28) % 21])
OUT3(EUCKR_JAMO_FIRSTBYTE)
OUT4(u2cgk_jongseong[c % 28])
NEXT(1, 4)
OUTBYTE1(EUCKR_JAMO_FIRSTBYTE)
OUTBYTE2(u2cgk_jungseong[(c / 28) % 21])
OUTBYTE3(EUCKR_JAMO_FIRSTBYTE)
OUTBYTE4(u2cgk_jongseong[c % 28])
NEXT(1, 4);
}
}
......@@ -102,7 +104,7 @@ static const unsigned char cgk2u_jongseong[] = { /* [A1, BE] */
DECODER(euc_kr)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
if (c < 0x80) {
OUTCHAR(c);
......@@ -113,7 +115,7 @@ DECODER(euc_kr)
REQUIRE_INBUF(2)
if (c == EUCKR_JAMO_FIRSTBYTE &&
IN2 == EUCKR_JAMO_FILLER) {
INBYTE2 == EUCKR_JAMO_FILLER) {
/* KS X 1001:1998 Annex 3 make-up sequence */
DBCHAR cho, jung, jong;
......@@ -146,7 +148,7 @@ DECODER(euc_kr)
OUTCHAR(0xac00 + cho*588 + jung*28 + jong);
NEXT_IN(8);
}
else TRYMAP_DEC(ksx1001, writer, c ^ 0x80, IN2 ^ 0x80) {
else TRYMAP_DEC(ksx1001, writer, c ^ 0x80, INBYTE2 ^ 0x80) {
NEXT_IN(2);
}
else
......@@ -164,27 +166,29 @@ DECODER(euc_kr)
ENCODER(cp949)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
TRYMAP_ENC(cp949, code, c);
else return 1;
OUT1((code >> 8) | 0x80)
OUTBYTE1((code >> 8) | 0x80)
if (code & 0x8000)
OUT2(code & 0xFF) /* MSB set: CP949 */
OUTBYTE2(code & 0xFF) /* MSB set: CP949 */
else
OUT2((code & 0xFF) | 0x80) /* MSB unset: ks x 1001 */
NEXT(1, 2)
OUTBYTE2((code & 0xFF) | 0x80) /* MSB unset: ks x 1001 */
NEXT(1, 2);
}
return 0;
......@@ -193,7 +197,7 @@ ENCODER(cp949)
DECODER(cp949)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
if (c < 0x80) {
OUTCHAR(c);
......@@ -202,8 +206,8 @@ DECODER(cp949)
}
REQUIRE_INBUF(2)
TRYMAP_DEC(ksx1001, writer, c ^ 0x80, IN2 ^ 0x80);
else TRYMAP_DEC(cp949ext, writer, c, IN2);
TRYMAP_DEC(ksx1001, writer, c ^ 0x80, INBYTE2 ^ 0x80);
else TRYMAP_DEC(cp949ext, writer, c, INBYTE2);
else return 1;
NEXT_IN(2);
......@@ -246,16 +250,18 @@ static const DBCHAR u2johabjamo[] = {
ENCODER(johab)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
......@@ -281,9 +287,9 @@ ENCODER(johab)
t1 = (c1 < 0x4a ? (c1 - 0x21 + 0x1b2) :
(c1 - 0x21 + 0x197));
t2 = ((t1 & 1) ? 0x5e : 0) + (c2 - 0x21);
OUT1(t1 >> 1)
OUT2(t2 < 0x4e ? t2 + 0x31 : t2 + 0x43)
NEXT(1, 2)
OUTBYTE1(t1 >> 1)
OUTBYTE2(t2 < 0x4e ? t2 + 0x31 : t2 + 0x43)
NEXT(1, 2);
continue;
}
else
......@@ -292,9 +298,9 @@ ENCODER(johab)
else
return 1;
OUT1(code >> 8)
OUT2(code & 0xff)
NEXT(1, 2)
OUTBYTE1(code >> 8)
OUTBYTE2(code & 0xff)
NEXT(1, 2);
}
return 0;
......@@ -344,7 +350,7 @@ static const unsigned char johabjamo_jongseong[32] = {
DECODER(johab)
{
while (inleft > 0) {
unsigned char c = IN1, c2;
unsigned char c = INBYTE1, c2;
if (c < 0x80) {
OUTCHAR(c);
......@@ -353,7 +359,7 @@ DECODER(johab)
}
REQUIRE_INBUF(2)
c2 = IN2;
c2 = INBYTE2;
if (c < 0xd8) {
/* johab hangul */
......
Modules/cjkcodecs/_codecs_tw.c
Dosyayı görüntüle @ d9491269
......@@ -13,26 +13,28 @@
ENCODER(big5)
{
while (inleft > 0) {
Py_UCS4 c = **inbuf;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
REQUIRE_OUTBUF(1)
**outbuf = (unsigned char)c;
NEXT(1, 1)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
TRYMAP_ENC(big5, code, c);
else return 1;
OUT1(code >> 8)
OUT2(code & 0xFF)
NEXT(1, 2)
OUTBYTE1(code >> 8)
OUTBYTE2(code & 0xFF)
NEXT(1, 2);
}
return 0;
......@@ -41,7 +43,7 @@ ENCODER(big5)
DECODER(big5)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
if (c < 0x80) {
OUTCHAR(c);
......@@ -50,7 +52,7 @@ DECODER(big5)
}
REQUIRE_INBUF(2)
TRYMAP_DEC(big5, writer, c, IN2) {
TRYMAP_DEC(big5, writer, c, INBYTE2) {
NEXT_IN(2);
}
else return 1;
......@@ -66,25 +68,27 @@ DECODER(big5)
ENCODER(cp950)
{
while (inleft > 0) {
Py_UCS4 c = IN1;
while (*inpos < inlen) {
Py_UCS4 c = INCHAR1;
DBCHAR code;
if (c < 0x80) {
WRITE1((unsigned char)c)
NEXT(1, 1)
WRITEBYTE1((unsigned char)c)
NEXT(1, 1);
continue;
}
UCS4INVALID(c)
if (c > 0xFFFF)
return 1;
REQUIRE_OUTBUF(2)
TRYMAP_ENC(cp950ext, code, c);
else TRYMAP_ENC(big5, code, c);
else return 1;
OUT1(code >> 8)
OUT2(code & 0xFF)
NEXT(1, 2)
OUTBYTE1(code >> 8)
OUTBYTE2(code & 0xFF)
NEXT(1, 2);
}
return 0;
......@@ -93,7 +97,7 @@ ENCODER(cp950)
DECODER(cp950)
{
while (inleft > 0) {
unsigned char c = IN1;
unsigned char c = INBYTE1;
if (c < 0x80) {
OUTCHAR(c);
......@@ -103,8 +107,8 @@ DECODER(cp950)
REQUIRE_INBUF(2)
TRYMAP_DEC(cp950ext, writer, c, IN2);
else TRYMAP_DEC(big5, writer, c, IN2);
TRYMAP_DEC(cp950ext, writer, c, INBYTE2);
else TRYMAP_DEC(big5, writer, c, INBYTE2);
else return 1;
NEXT_IN(2);
......
Modules/cjkcodecs/cjkcodecs.h
Dosyayı görüntüle @ d9491269
......@@ -72,7 +72,8 @@ static const struct dbcs_map *mapping_list;
#define ENCODER(encoding) \
static Py_ssize_t encoding##_encode( \
MultibyteCodec_State *state, const void *config, \
const Py_UNICODE **inbuf, Py_ssize_t inleft, \
int kind, void *data, \
Py_ssize_t *inpos, Py_ssize_t inlen, \
unsigned char **outbuf, Py_ssize_t outleft, int flags)
#define ENCODER_RESET(encoding) \
static Py_ssize_t encoding##_encode_reset( \
......@@ -91,25 +92,25 @@ static const struct dbcs_map *mapping_list;
static Py_ssize_t encoding##_decode_reset( \
MultibyteCodec_State *state, const void *config)
#if Py_UNICODE_SIZE == 4
#define UCS4INVALID(code) \
if ((code) > 0xFFFF) \
return 1;
#else
#define UCS4INVALID(code) \
if (0) ;
#endif
#define NEXT_IN(i) \
do { \
(*inbuf) += (i); \
(inleft) -= (i); \
} while (0)
#define NEXT_INCHAR(i) \
do { \
(*inpos) += (i); \
} while (0)
#define NEXT_OUT(o) \
do { \
(*outbuf) += (o); \
(outleft) -= (o);
(outleft) -= (o); \
} while (0)
#define NEXT(i, o) \
NEXT_IN(i); NEXT_OUT(o)
do { \
NEXT_INCHAR(i); \
NEXT_OUT(o); \
} while (0)
#define REQUIRE_INBUF(n) \
if (inleft < (n)) \
......@@ -118,10 +119,13 @@ static const struct dbcs_map *mapping_list;
if (outleft < (n)) \
return MBERR_TOOSMALL;
#define IN1 ((*inbuf)[0])
#define IN2 ((*inbuf)[1])
#define IN3 ((*inbuf)[2])
#define IN4 ((*inbuf)[3])
#define INBYTE1 ((*inbuf)[0])
#define INBYTE2 ((*inbuf)[1])
#define INBYTE3 ((*inbuf)[2])
#define INBYTE4 ((*inbuf)[3])
#define INCHAR1 PyUnicode_READ(kind, data, *inpos)
#define INCHAR2 PyUnicode_READ(kind, data, *inpos + 1)
#define OUTCHAR(c) \
do { \
......@@ -140,24 +144,24 @@ static const struct dbcs_map *mapping_list;
writer->pos += 2; \
} while (0)
#define OUT1(c) ((*outbuf)[0]) = (c);
#define OUT2(c) ((*outbuf)[1]) = (c);
#define OUT3(c) ((*outbuf)[2]) = (c);
#define OUT4(c) ((*outbuf)[3]) = (c);
#define OUTBYTE1(c) ((*outbuf)[0]) = (c);
#define OUTBYTE2(c) ((*outbuf)[1]) = (c);
#define OUTBYTE3(c) ((*outbuf)[2]) = (c);
#define OUTBYTE4(c) ((*outbuf)[3]) = (c);
#define WRITE1(c1) \
#define WRITEBYTE1(c1) \
REQUIRE_OUTBUF(1) \
(*outbuf)[0] = (c1);
#define WRITE2(c1, c2) \
#define WRITEBYTE2(c1, c2) \
REQUIRE_OUTBUF(2) \
(*outbuf)[0] = (c1); \
(*outbuf)[1] = (c2);
#define WRITE3(c1, c2, c3) \
#define WRITEBYTE3(c1, c2, c3) \
REQUIRE_OUTBUF(3) \
(*outbuf)[0] = (c1); \
(*outbuf)[1] = (c2); \
(*outbuf)[2] = (c3);
#define WRITE4(c1, c2, c3, c4) \
#define WRITEBYTE4(c1, c2, c3, c4) \
REQUIRE_OUTBUF(4) \
(*outbuf)[0] = (c1); \
(*outbuf)[1] = (c2); \
......@@ -209,20 +213,6 @@ _TRYMAP_DEC_WRITE(_PyUnicodeWriter *writer, Py_UCS4 c)
#define TRYMAP_DEC_MPLANE(charset, writer, plane, c1, c2) \
if _TRYMAP_DEC(&charset##_decmap[plane][c1], writer, c2)
#if Py_UNICODE_SIZE == 2
#define DECODE_SURROGATE(c) \
if (Py_UNICODE_IS_HIGH_SURROGATE(c)) { \
REQUIRE_INBUF(2) \
if (Py_UNICODE_IS_LOW_SURROGATE(IN2)) { \
c = Py_UNICODE_JOIN_SURROGATES(c, IN2); \
} \
}
#define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1)
#else
#define DECODE_SURROGATE(c) {;}
#define GET_INSIZE(c) 1
#endif
#define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = {
#define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
#define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
......
Modules/cjkcodecs/multibytecodec.h
Dosyayı görüntüle @ d9491269
......@@ -27,7 +27,8 @@ typedef union {
typedef int (*mbcodec_init)(const void *config);
typedef Py_ssize_t (*mbencode_func)(MultibyteCodec_State *state,
const void *config,
const Py_UNICODE **inbuf, Py_ssize_t inleft,
int kind, void *data,
Py_ssize_t *inpos, Py_ssize_t inlen,
unsigned char **outbuf, Py_ssize_t outleft,
int flags);
typedef int (*mbencodeinit_func)(MultibyteCodec_State *state,
......@@ -75,8 +76,7 @@ typedef struct {
#define MAXENCPENDING 2
#define _MultibyteStatefulEncoder_HEAD \
_MultibyteStatefulCodec_HEAD \
Py_UNICODE pending[MAXENCPENDING]; \
Py_ssize_t pendingsize;
PyObject *pending;
typedef struct {
_MultibyteStatefulEncoder_HEAD
} MultibyteStatefulEncoderContext;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment