2133 lines
45 KiB
C
2133 lines
45 KiB
C
/*-------------------------------------------------------------------------
|
|
*
|
|
* wchar.c
|
|
* Functions for working with multibyte characters in various encodings.
|
|
*
|
|
* Portions Copyright (c) 1998-2024, PostgreSQL Global Development Group
|
|
*
|
|
* IDENTIFICATION
|
|
* src/common/wchar.c
|
|
*
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
#include "c.h"
|
|
|
|
#include "mb/pg_wchar.h"
|
|
#include "utils/ascii.h"
|
|
|
|
|
|
/*
|
|
* Operations on multi-byte encodings are driven by a table of helper
|
|
* functions.
|
|
*
|
|
* To add an encoding support, define mblen(), dsplen(), verifychar() and
|
|
* verifystr() for the encoding. For server-encodings, also define mb2wchar()
|
|
* and wchar2mb() conversion functions.
|
|
*
|
|
* These functions generally assume that their input is validly formed.
|
|
* The "verifier" functions, further down in the file, have to be more
|
|
* paranoid.
|
|
*
|
|
* We expect that mblen() does not need to examine more than the first byte
|
|
* of the character to discover the correct length. GB18030 is an exception
|
|
* to that rule, though, as it also looks at second byte. But even that
|
|
* behaves in a predictable way, if you only pass the first byte: it will
|
|
* treat 4-byte encoded characters as two 2-byte encoded characters, which is
|
|
* good enough for all current uses.
|
|
*
|
|
* Note: for the display output of psql to work properly, the return values
|
|
* of the dsplen functions must conform to the Unicode standard. In particular
|
|
* the NUL character is zero width and control characters are generally
|
|
* width -1. It is recommended that non-ASCII encodings refer their ASCII
|
|
* subset to the ASCII routines to ensure consistency.
|
|
*/
|
|
|
|
/*
|
|
* SQL/ASCII
|
|
*/
|
|
static int
|
|
pg_ascii2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
*to++ = *from++;
|
|
len--;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
static int
|
|
pg_ascii_mblen(const unsigned char *s)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
static int
|
|
pg_ascii_dsplen(const unsigned char *s)
|
|
{
|
|
if (*s == '\0')
|
|
return 0;
|
|
if (*s < 0x20 || *s == 0x7f)
|
|
return -1;
|
|
|
|
return 1;
|
|
}
|
|
|
|
/*
|
|
* EUC
|
|
*/
|
|
static int
|
|
pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
if (*from == SS2 && len >= 2) /* JIS X 0201 (so called "1 byte
|
|
* KANA") */
|
|
{
|
|
from++;
|
|
*to = (SS2 << 8) | *from++;
|
|
len -= 2;
|
|
}
|
|
else if (*from == SS3 && len >= 3) /* JIS X 0212 KANJI */
|
|
{
|
|
from++;
|
|
*to = (SS3 << 16) | (*from++ << 8);
|
|
*to |= *from++;
|
|
len -= 3;
|
|
}
|
|
else if (IS_HIGHBIT_SET(*from) && len >= 2) /* JIS X 0208 KANJI */
|
|
{
|
|
*to = *from++ << 8;
|
|
*to |= *from++;
|
|
len -= 2;
|
|
}
|
|
else /* must be ASCII */
|
|
{
|
|
*to = *from++;
|
|
len--;
|
|
}
|
|
to++;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
static inline int
|
|
pg_euc_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s == SS2)
|
|
len = 2;
|
|
else if (*s == SS3)
|
|
len = 3;
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = 1;
|
|
return len;
|
|
}
|
|
|
|
static inline int
|
|
pg_euc_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s == SS2)
|
|
len = 2;
|
|
else if (*s == SS3)
|
|
len = 2;
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = pg_ascii_dsplen(s);
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* EUC_JP
|
|
*/
|
|
static int
|
|
pg_eucjp2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
return pg_euc2wchar_with_len(from, to, len);
|
|
}
|
|
|
|
static int
|
|
pg_eucjp_mblen(const unsigned char *s)
|
|
{
|
|
return pg_euc_mblen(s);
|
|
}
|
|
|
|
static int
|
|
pg_eucjp_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s == SS2)
|
|
len = 1;
|
|
else if (*s == SS3)
|
|
len = 2;
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = pg_ascii_dsplen(s);
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* EUC_KR
|
|
*/
|
|
static int
|
|
pg_euckr2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
return pg_euc2wchar_with_len(from, to, len);
|
|
}
|
|
|
|
static int
|
|
pg_euckr_mblen(const unsigned char *s)
|
|
{
|
|
return pg_euc_mblen(s);
|
|
}
|
|
|
|
static int
|
|
pg_euckr_dsplen(const unsigned char *s)
|
|
{
|
|
return pg_euc_dsplen(s);
|
|
}
|
|
|
|
/*
|
|
* EUC_CN
|
|
*
|
|
*/
|
|
static int
|
|
pg_euccn2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
if (*from == SS2 && len >= 3) /* code set 2 (unused?) */
|
|
{
|
|
from++;
|
|
*to = (SS2 << 16) | (*from++ << 8);
|
|
*to |= *from++;
|
|
len -= 3;
|
|
}
|
|
else if (*from == SS3 && len >= 3) /* code set 3 (unused ?) */
|
|
{
|
|
from++;
|
|
*to = (SS3 << 16) | (*from++ << 8);
|
|
*to |= *from++;
|
|
len -= 3;
|
|
}
|
|
else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 1 */
|
|
{
|
|
*to = *from++ << 8;
|
|
*to |= *from++;
|
|
len -= 2;
|
|
}
|
|
else
|
|
{
|
|
*to = *from++;
|
|
len--;
|
|
}
|
|
to++;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
static int
|
|
pg_euccn_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = 1;
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_euccn_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = pg_ascii_dsplen(s);
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* EUC_TW
|
|
*
|
|
*/
|
|
static int
|
|
pg_euctw2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
if (*from == SS2 && len >= 4) /* code set 2 */
|
|
{
|
|
from++;
|
|
*to = (((uint32) SS2) << 24) | (*from++ << 16);
|
|
*to |= *from++ << 8;
|
|
*to |= *from++;
|
|
len -= 4;
|
|
}
|
|
else if (*from == SS3 && len >= 3) /* code set 3 (unused?) */
|
|
{
|
|
from++;
|
|
*to = (SS3 << 16) | (*from++ << 8);
|
|
*to |= *from++;
|
|
len -= 3;
|
|
}
|
|
else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 2 */
|
|
{
|
|
*to = *from++ << 8;
|
|
*to |= *from++;
|
|
len -= 2;
|
|
}
|
|
else
|
|
{
|
|
*to = *from++;
|
|
len--;
|
|
}
|
|
to++;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
static int
|
|
pg_euctw_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s == SS2)
|
|
len = 4;
|
|
else if (*s == SS3)
|
|
len = 3;
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = 1;
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_euctw_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s == SS2)
|
|
len = 2;
|
|
else if (*s == SS3)
|
|
len = 2;
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = pg_ascii_dsplen(s);
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* Convert pg_wchar to EUC_* encoding.
|
|
* caller must allocate enough space for "to", including a trailing zero!
|
|
* len: length of from.
|
|
* "from" not necessarily null terminated.
|
|
*/
|
|
static int
|
|
pg_wchar2euc_with_len(const pg_wchar *from, unsigned char *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
unsigned char c;
|
|
|
|
if ((c = (*from >> 24)))
|
|
{
|
|
*to++ = c;
|
|
*to++ = (*from >> 16) & 0xff;
|
|
*to++ = (*from >> 8) & 0xff;
|
|
*to++ = *from & 0xff;
|
|
cnt += 4;
|
|
}
|
|
else if ((c = (*from >> 16)))
|
|
{
|
|
*to++ = c;
|
|
*to++ = (*from >> 8) & 0xff;
|
|
*to++ = *from & 0xff;
|
|
cnt += 3;
|
|
}
|
|
else if ((c = (*from >> 8)))
|
|
{
|
|
*to++ = c;
|
|
*to++ = *from & 0xff;
|
|
cnt += 2;
|
|
}
|
|
else
|
|
{
|
|
*to++ = *from;
|
|
cnt++;
|
|
}
|
|
from++;
|
|
len--;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
|
|
/*
|
|
* JOHAB
|
|
*/
|
|
static int
|
|
pg_johab_mblen(const unsigned char *s)
|
|
{
|
|
return pg_euc_mblen(s);
|
|
}
|
|
|
|
static int
|
|
pg_johab_dsplen(const unsigned char *s)
|
|
{
|
|
return pg_euc_dsplen(s);
|
|
}
|
|
|
|
/*
|
|
* convert UTF8 string to pg_wchar (UCS-4)
|
|
* caller must allocate enough space for "to", including a trailing zero!
|
|
* len: length of from.
|
|
* "from" not necessarily null terminated.
|
|
*/
|
|
static int
|
|
pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
uint32 c1,
|
|
c2,
|
|
c3,
|
|
c4;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
if ((*from & 0x80) == 0)
|
|
{
|
|
*to = *from++;
|
|
len--;
|
|
}
|
|
else if ((*from & 0xe0) == 0xc0)
|
|
{
|
|
if (len < 2)
|
|
break; /* drop trailing incomplete char */
|
|
c1 = *from++ & 0x1f;
|
|
c2 = *from++ & 0x3f;
|
|
*to = (c1 << 6) | c2;
|
|
len -= 2;
|
|
}
|
|
else if ((*from & 0xf0) == 0xe0)
|
|
{
|
|
if (len < 3)
|
|
break; /* drop trailing incomplete char */
|
|
c1 = *from++ & 0x0f;
|
|
c2 = *from++ & 0x3f;
|
|
c3 = *from++ & 0x3f;
|
|
*to = (c1 << 12) | (c2 << 6) | c3;
|
|
len -= 3;
|
|
}
|
|
else if ((*from & 0xf8) == 0xf0)
|
|
{
|
|
if (len < 4)
|
|
break; /* drop trailing incomplete char */
|
|
c1 = *from++ & 0x07;
|
|
c2 = *from++ & 0x3f;
|
|
c3 = *from++ & 0x3f;
|
|
c4 = *from++ & 0x3f;
|
|
*to = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
|
|
len -= 4;
|
|
}
|
|
else
|
|
{
|
|
/* treat a bogus char as length 1; not ours to raise error */
|
|
*to = *from++;
|
|
len--;
|
|
}
|
|
to++;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
|
|
/*
|
|
* Trivial conversion from pg_wchar to UTF-8.
|
|
* caller should allocate enough space for "to"
|
|
* len: length of from.
|
|
* "from" not necessarily null terminated.
|
|
*/
|
|
static int
|
|
pg_wchar2utf_with_len(const pg_wchar *from, unsigned char *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
int char_len;
|
|
|
|
unicode_to_utf8(*from, to);
|
|
char_len = pg_utf_mblen(to);
|
|
cnt += char_len;
|
|
to += char_len;
|
|
from++;
|
|
len--;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
/*
|
|
* Return the byte length of a UTF8 character pointed to by s
|
|
*
|
|
* Note: in the current implementation we do not support UTF8 sequences
|
|
* of more than 4 bytes; hence do NOT return a value larger than 4.
|
|
* We return "1" for any leading byte that is either flat-out illegal or
|
|
* indicates a length larger than we support.
|
|
*
|
|
* pg_utf2wchar_with_len(), utf8_to_unicode(), pg_utf8_islegal(), and perhaps
|
|
* other places would need to be fixed to change this.
|
|
*/
|
|
int
|
|
pg_utf_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if ((*s & 0x80) == 0)
|
|
len = 1;
|
|
else if ((*s & 0xe0) == 0xc0)
|
|
len = 2;
|
|
else if ((*s & 0xf0) == 0xe0)
|
|
len = 3;
|
|
else if ((*s & 0xf8) == 0xf0)
|
|
len = 4;
|
|
#ifdef NOT_USED
|
|
else if ((*s & 0xfc) == 0xf8)
|
|
len = 5;
|
|
else if ((*s & 0xfe) == 0xfc)
|
|
len = 6;
|
|
#endif
|
|
else
|
|
len = 1;
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* This is an implementation of wcwidth() and wcswidth() as defined in
|
|
* "The Single UNIX Specification, Version 2, The Open Group, 1997"
|
|
* <http://www.unix.org/online.html>
|
|
*
|
|
* Markus Kuhn -- 2001-09-08 -- public domain
|
|
*
|
|
* customised for PostgreSQL
|
|
*
|
|
* original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
|
|
*/
|
|
|
|
struct mbinterval
|
|
{
|
|
unsigned int first;
|
|
unsigned int last;
|
|
};
|
|
|
|
/* auxiliary function for binary search in interval table */
|
|
static int
|
|
mbbisearch(pg_wchar ucs, const struct mbinterval *table, int max)
|
|
{
|
|
int min = 0;
|
|
int mid;
|
|
|
|
if (ucs < table[0].first || ucs > table[max].last)
|
|
return 0;
|
|
while (max >= min)
|
|
{
|
|
mid = (min + max) / 2;
|
|
if (ucs > table[mid].last)
|
|
min = mid + 1;
|
|
else if (ucs < table[mid].first)
|
|
max = mid - 1;
|
|
else
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
|
|
/* The following functions define the column width of an ISO 10646
|
|
* character as follows:
|
|
*
|
|
* - The null character (U+0000) has a column width of 0.
|
|
*
|
|
* - Other C0/C1 control characters and DEL will lead to a return
|
|
* value of -1.
|
|
*
|
|
* - Non-spacing and enclosing combining characters (general
|
|
* category code Mn, Me or Cf in the Unicode database) have a
|
|
* column width of 0.
|
|
*
|
|
* - Spacing characters in the East Asian Wide (W) or East Asian
|
|
* FullWidth (F) category as defined in Unicode Technical
|
|
* Report #11 have a column width of 2.
|
|
*
|
|
* - All remaining characters (including all printable
|
|
* ISO 8859-1 and WGL4 characters, Unicode control characters,
|
|
* etc.) have a column width of 1.
|
|
*
|
|
* This implementation assumes that wchar_t characters are encoded
|
|
* in ISO 10646.
|
|
*/
|
|
|
|
static int
|
|
ucs_wcwidth(pg_wchar ucs)
|
|
{
|
|
#include "common/unicode_nonspacing_table.h"
|
|
#include "common/unicode_east_asian_fw_table.h"
|
|
|
|
/* test for 8-bit control characters */
|
|
if (ucs == 0)
|
|
return 0;
|
|
|
|
if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
|
|
return -1;
|
|
|
|
/*
|
|
* binary search in table of non-spacing characters
|
|
*
|
|
* XXX: In the official Unicode sources, it is possible for a character to
|
|
* be described as both non-spacing and wide at the same time. As of
|
|
* Unicode 13.0, treating the non-spacing property as the determining
|
|
* factor for display width leads to the correct behavior, so do that
|
|
* search first.
|
|
*/
|
|
if (mbbisearch(ucs, nonspacing,
|
|
sizeof(nonspacing) / sizeof(struct mbinterval) - 1))
|
|
return 0;
|
|
|
|
/* binary search in table of wide characters */
|
|
if (mbbisearch(ucs, east_asian_fw,
|
|
sizeof(east_asian_fw) / sizeof(struct mbinterval) - 1))
|
|
return 2;
|
|
|
|
return 1;
|
|
}
|
|
|
|
static int
|
|
pg_utf_dsplen(const unsigned char *s)
|
|
{
|
|
return ucs_wcwidth(utf8_to_unicode(s));
|
|
}
|
|
|
|
/*
|
|
* convert mule internal code to pg_wchar
|
|
* caller should allocate enough space for "to"
|
|
* len: length of from.
|
|
* "from" not necessarily null terminated.
|
|
*/
|
|
static int
|
|
pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
if (IS_LC1(*from) && len >= 2)
|
|
{
|
|
*to = *from++ << 16;
|
|
*to |= *from++;
|
|
len -= 2;
|
|
}
|
|
else if (IS_LCPRV1(*from) && len >= 3)
|
|
{
|
|
from++;
|
|
*to = *from++ << 16;
|
|
*to |= *from++;
|
|
len -= 3;
|
|
}
|
|
else if (IS_LC2(*from) && len >= 3)
|
|
{
|
|
*to = *from++ << 16;
|
|
*to |= *from++ << 8;
|
|
*to |= *from++;
|
|
len -= 3;
|
|
}
|
|
else if (IS_LCPRV2(*from) && len >= 4)
|
|
{
|
|
from++;
|
|
*to = *from++ << 16;
|
|
*to |= *from++ << 8;
|
|
*to |= *from++;
|
|
len -= 4;
|
|
}
|
|
else
|
|
{ /* assume ASCII */
|
|
*to = (unsigned char) *from++;
|
|
len--;
|
|
}
|
|
to++;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
/*
|
|
* convert pg_wchar to mule internal code
|
|
* caller should allocate enough space for "to"
|
|
* len: length of from.
|
|
* "from" not necessarily null terminated.
|
|
*/
|
|
static int
|
|
pg_wchar2mule_with_len(const pg_wchar *from, unsigned char *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
unsigned char lb;
|
|
|
|
lb = (*from >> 16) & 0xff;
|
|
if (IS_LC1(lb))
|
|
{
|
|
*to++ = lb;
|
|
*to++ = *from & 0xff;
|
|
cnt += 2;
|
|
}
|
|
else if (IS_LC2(lb))
|
|
{
|
|
*to++ = lb;
|
|
*to++ = (*from >> 8) & 0xff;
|
|
*to++ = *from & 0xff;
|
|
cnt += 3;
|
|
}
|
|
else if (IS_LCPRV1_A_RANGE(lb))
|
|
{
|
|
*to++ = LCPRV1_A;
|
|
*to++ = lb;
|
|
*to++ = *from & 0xff;
|
|
cnt += 3;
|
|
}
|
|
else if (IS_LCPRV1_B_RANGE(lb))
|
|
{
|
|
*to++ = LCPRV1_B;
|
|
*to++ = lb;
|
|
*to++ = *from & 0xff;
|
|
cnt += 3;
|
|
}
|
|
else if (IS_LCPRV2_A_RANGE(lb))
|
|
{
|
|
*to++ = LCPRV2_A;
|
|
*to++ = lb;
|
|
*to++ = (*from >> 8) & 0xff;
|
|
*to++ = *from & 0xff;
|
|
cnt += 4;
|
|
}
|
|
else if (IS_LCPRV2_B_RANGE(lb))
|
|
{
|
|
*to++ = LCPRV2_B;
|
|
*to++ = lb;
|
|
*to++ = (*from >> 8) & 0xff;
|
|
*to++ = *from & 0xff;
|
|
cnt += 4;
|
|
}
|
|
else
|
|
{
|
|
*to++ = *from & 0xff;
|
|
cnt += 1;
|
|
}
|
|
from++;
|
|
len--;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
/* exported for direct use by conv.c */
|
|
int
|
|
pg_mule_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_LC1(*s))
|
|
len = 2;
|
|
else if (IS_LCPRV1(*s))
|
|
len = 3;
|
|
else if (IS_LC2(*s))
|
|
len = 3;
|
|
else if (IS_LCPRV2(*s))
|
|
len = 4;
|
|
else
|
|
len = 1; /* assume ASCII */
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_mule_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
/*
|
|
* Note: it's not really appropriate to assume that all multibyte charsets
|
|
* are double-wide on screen. But this seems an okay approximation for
|
|
* the MULE charsets we currently support.
|
|
*/
|
|
|
|
if (IS_LC1(*s))
|
|
len = 1;
|
|
else if (IS_LCPRV1(*s))
|
|
len = 1;
|
|
else if (IS_LC2(*s))
|
|
len = 2;
|
|
else if (IS_LCPRV2(*s))
|
|
len = 2;
|
|
else
|
|
len = 1; /* assume ASCII */
|
|
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* ISO8859-1
|
|
*/
|
|
static int
|
|
pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
*to++ = *from++;
|
|
len--;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
/*
|
|
* Trivial conversion from pg_wchar to single byte encoding. Just ignores
|
|
* high bits.
|
|
* caller should allocate enough space for "to"
|
|
* len: length of from.
|
|
* "from" not necessarily null terminated.
|
|
*/
|
|
static int
|
|
pg_wchar2single_with_len(const pg_wchar *from, unsigned char *to, int len)
|
|
{
|
|
int cnt = 0;
|
|
|
|
while (len > 0 && *from)
|
|
{
|
|
*to++ = *from++;
|
|
len--;
|
|
cnt++;
|
|
}
|
|
*to = 0;
|
|
return cnt;
|
|
}
|
|
|
|
static int
|
|
pg_latin1_mblen(const unsigned char *s)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
static int
|
|
pg_latin1_dsplen(const unsigned char *s)
|
|
{
|
|
return pg_ascii_dsplen(s);
|
|
}
|
|
|
|
/*
|
|
* SJIS
|
|
*/
|
|
static int
|
|
pg_sjis_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s >= 0xa1 && *s <= 0xdf)
|
|
len = 1; /* 1 byte kana? */
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* kanji? */
|
|
else
|
|
len = 1; /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_sjis_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (*s >= 0xa1 && *s <= 0xdf)
|
|
len = 1; /* 1 byte kana? */
|
|
else if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* kanji? */
|
|
else
|
|
len = pg_ascii_dsplen(s); /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* Big5
|
|
*/
|
|
static int
|
|
pg_big5_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* kanji? */
|
|
else
|
|
len = 1; /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_big5_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* kanji? */
|
|
else
|
|
len = pg_ascii_dsplen(s); /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* GBK
|
|
*/
|
|
static int
|
|
pg_gbk_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* kanji? */
|
|
else
|
|
len = 1; /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_gbk_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* kanji? */
|
|
else
|
|
len = pg_ascii_dsplen(s); /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* UHC
|
|
*/
|
|
static int
|
|
pg_uhc_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* 2byte? */
|
|
else
|
|
len = 1; /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_uhc_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2; /* 2byte? */
|
|
else
|
|
len = pg_ascii_dsplen(s); /* should be ASCII */
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* GB18030
|
|
* Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
|
|
*/
|
|
|
|
/*
|
|
* Unlike all other mblen() functions, this also looks at the second byte of
|
|
* the input. However, if you only pass the first byte of a multi-byte
|
|
* string, and \0 as the second byte, this still works in a predictable way:
|
|
* a 4-byte character will be reported as two 2-byte characters. That's
|
|
* enough for all current uses, as a client-only encoding. It works that
|
|
* way, because in any valid 4-byte GB18030-encoded character, the third and
|
|
* fourth byte look like a 2-byte encoded character, when looked at
|
|
* separately.
|
|
*/
|
|
static int
|
|
pg_gb18030_mblen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
len = 1; /* ASCII */
|
|
else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
|
|
len = 4;
|
|
else
|
|
len = 2;
|
|
return len;
|
|
}
|
|
|
|
static int
|
|
pg_gb18030_dsplen(const unsigned char *s)
|
|
{
|
|
int len;
|
|
|
|
if (IS_HIGHBIT_SET(*s))
|
|
len = 2;
|
|
else
|
|
len = pg_ascii_dsplen(s); /* ASCII */
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
*-------------------------------------------------------------------
|
|
* multibyte sequence validators
|
|
*
|
|
* The verifychar functions accept "s", a pointer to the first byte of a
|
|
* string, and "len", the remaining length of the string. If there is a
|
|
* validly encoded character beginning at *s, return its length in bytes;
|
|
* else return -1.
|
|
*
|
|
* The verifystr functions also accept "s", a pointer to a string and "len",
|
|
* the length of the string. They verify the whole string, and return the
|
|
* number of input bytes (<= len) that are valid. In other words, if the
|
|
* whole string is valid, verifystr returns "len", otherwise it returns the
|
|
* byte offset of the first invalid character. The verifystr functions must
|
|
* test for and reject zeroes in the input.
|
|
*
|
|
* The verifychar functions can assume that len > 0 and that *s != '\0', but
|
|
* they must test for and reject zeroes in any additional bytes of a
|
|
* multibyte character. Note that this definition allows the function for a
|
|
* single-byte encoding to be just "return 1".
|
|
*-------------------------------------------------------------------
|
|
*/
|
|
static int
|
|
pg_ascii_verifychar(const unsigned char *s, int len)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
static int
|
|
pg_ascii_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *nullpos = memchr(s, 0, len);
|
|
|
|
if (nullpos == NULL)
|
|
return len;
|
|
else
|
|
return nullpos - s;
|
|
}
|
|
|
|
#define IS_EUC_RANGE_VALID(c) ((c) >= 0xa1 && (c) <= 0xfe)
|
|
|
|
static int
|
|
pg_eucjp_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l;
|
|
unsigned char c1,
|
|
c2;
|
|
|
|
c1 = *s++;
|
|
|
|
switch (c1)
|
|
{
|
|
case SS2: /* JIS X 0201 */
|
|
l = 2;
|
|
if (l > len)
|
|
return -1;
|
|
c2 = *s++;
|
|
if (c2 < 0xa1 || c2 > 0xdf)
|
|
return -1;
|
|
break;
|
|
|
|
case SS3: /* JIS X 0212 */
|
|
l = 3;
|
|
if (l > len)
|
|
return -1;
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
break;
|
|
|
|
default:
|
|
if (IS_HIGHBIT_SET(c1)) /* JIS X 0208? */
|
|
{
|
|
l = 2;
|
|
if (l > len)
|
|
return -1;
|
|
if (!IS_EUC_RANGE_VALID(c1))
|
|
return -1;
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
}
|
|
else
|
|
/* must be ASCII */
|
|
{
|
|
l = 1;
|
|
}
|
|
break;
|
|
}
|
|
|
|
return l;
|
|
}
|
|
|
|
static int
|
|
pg_eucjp_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_eucjp_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_euckr_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l;
|
|
unsigned char c1,
|
|
c2;
|
|
|
|
c1 = *s++;
|
|
|
|
if (IS_HIGHBIT_SET(c1))
|
|
{
|
|
l = 2;
|
|
if (l > len)
|
|
return -1;
|
|
if (!IS_EUC_RANGE_VALID(c1))
|
|
return -1;
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
}
|
|
else
|
|
/* must be ASCII */
|
|
{
|
|
l = 1;
|
|
}
|
|
|
|
return l;
|
|
}
|
|
|
|
static int
|
|
pg_euckr_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_euckr_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
/* EUC-CN byte sequences are exactly same as EUC-KR */
|
|
#define pg_euccn_verifychar pg_euckr_verifychar
|
|
#define pg_euccn_verifystr pg_euckr_verifystr
|
|
|
|
static int
|
|
pg_euctw_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l;
|
|
unsigned char c1,
|
|
c2;
|
|
|
|
c1 = *s++;
|
|
|
|
switch (c1)
|
|
{
|
|
case SS2: /* CNS 11643 Plane 1-7 */
|
|
l = 4;
|
|
if (l > len)
|
|
return -1;
|
|
c2 = *s++;
|
|
if (c2 < 0xa1 || c2 > 0xa7)
|
|
return -1;
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
break;
|
|
|
|
case SS3: /* unused */
|
|
return -1;
|
|
|
|
default:
|
|
if (IS_HIGHBIT_SET(c1)) /* CNS 11643 Plane 1 */
|
|
{
|
|
l = 2;
|
|
if (l > len)
|
|
return -1;
|
|
/* no further range check on c1? */
|
|
c2 = *s++;
|
|
if (!IS_EUC_RANGE_VALID(c2))
|
|
return -1;
|
|
}
|
|
else
|
|
/* must be ASCII */
|
|
{
|
|
l = 1;
|
|
}
|
|
break;
|
|
}
|
|
return l;
|
|
}
|
|
|
|
static int
|
|
pg_euctw_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_euctw_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_johab_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l,
|
|
mbl;
|
|
unsigned char c;
|
|
|
|
l = mbl = pg_johab_mblen(s);
|
|
|
|
if (len < l)
|
|
return -1;
|
|
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
return mbl;
|
|
|
|
while (--l > 0)
|
|
{
|
|
c = *++s;
|
|
if (!IS_EUC_RANGE_VALID(c))
|
|
return -1;
|
|
}
|
|
return mbl;
|
|
}
|
|
|
|
static int
|
|
pg_johab_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_johab_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_mule_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l,
|
|
mbl;
|
|
unsigned char c;
|
|
|
|
l = mbl = pg_mule_mblen(s);
|
|
|
|
if (len < l)
|
|
return -1;
|
|
|
|
while (--l > 0)
|
|
{
|
|
c = *++s;
|
|
if (!IS_HIGHBIT_SET(c))
|
|
return -1;
|
|
}
|
|
return mbl;
|
|
}
|
|
|
|
static int
|
|
pg_mule_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_mule_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_latin1_verifychar(const unsigned char *s, int len)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
static int
|
|
pg_latin1_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *nullpos = memchr(s, 0, len);
|
|
|
|
if (nullpos == NULL)
|
|
return len;
|
|
else
|
|
return nullpos - s;
|
|
}
|
|
|
|
static int
|
|
pg_sjis_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l,
|
|
mbl;
|
|
unsigned char c1,
|
|
c2;
|
|
|
|
l = mbl = pg_sjis_mblen(s);
|
|
|
|
if (len < l)
|
|
return -1;
|
|
|
|
if (l == 1) /* pg_sjis_mblen already verified it */
|
|
return mbl;
|
|
|
|
c1 = *s++;
|
|
c2 = *s;
|
|
if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
|
|
return -1;
|
|
return mbl;
|
|
}
|
|
|
|
static int
|
|
pg_sjis_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_sjis_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_big5_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l,
|
|
mbl;
|
|
|
|
l = mbl = pg_big5_mblen(s);
|
|
|
|
if (len < l)
|
|
return -1;
|
|
|
|
while (--l > 0)
|
|
{
|
|
if (*++s == '\0')
|
|
return -1;
|
|
}
|
|
|
|
return mbl;
|
|
}
|
|
|
|
static int
|
|
pg_big5_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_big5_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_gbk_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l,
|
|
mbl;
|
|
|
|
l = mbl = pg_gbk_mblen(s);
|
|
|
|
if (len < l)
|
|
return -1;
|
|
|
|
while (--l > 0)
|
|
{
|
|
if (*++s == '\0')
|
|
return -1;
|
|
}
|
|
|
|
return mbl;
|
|
}
|
|
|
|
static int
|
|
pg_gbk_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_gbk_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_uhc_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l,
|
|
mbl;
|
|
|
|
l = mbl = pg_uhc_mblen(s);
|
|
|
|
if (len < l)
|
|
return -1;
|
|
|
|
while (--l > 0)
|
|
{
|
|
if (*++s == '\0')
|
|
return -1;
|
|
}
|
|
|
|
return mbl;
|
|
}
|
|
|
|
static int
|
|
pg_uhc_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_uhc_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_gb18030_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l;
|
|
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
l = 1; /* ASCII */
|
|
else if (len >= 4 && *(s + 1) >= 0x30 && *(s + 1) <= 0x39)
|
|
{
|
|
/* Should be 4-byte, validate remaining bytes */
|
|
if (*s >= 0x81 && *s <= 0xfe &&
|
|
*(s + 2) >= 0x81 && *(s + 2) <= 0xfe &&
|
|
*(s + 3) >= 0x30 && *(s + 3) <= 0x39)
|
|
l = 4;
|
|
else
|
|
l = -1;
|
|
}
|
|
else if (len >= 2 && *s >= 0x81 && *s <= 0xfe)
|
|
{
|
|
/* Should be 2-byte, validate */
|
|
if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) ||
|
|
(*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
|
|
l = 2;
|
|
else
|
|
l = -1;
|
|
}
|
|
else
|
|
l = -1;
|
|
return l;
|
|
}
|
|
|
|
static int
|
|
pg_gb18030_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_gb18030_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
static int
|
|
pg_utf8_verifychar(const unsigned char *s, int len)
|
|
{
|
|
int l;
|
|
|
|
if ((*s & 0x80) == 0)
|
|
{
|
|
if (*s == '\0')
|
|
return -1;
|
|
return 1;
|
|
}
|
|
else if ((*s & 0xe0) == 0xc0)
|
|
l = 2;
|
|
else if ((*s & 0xf0) == 0xe0)
|
|
l = 3;
|
|
else if ((*s & 0xf8) == 0xf0)
|
|
l = 4;
|
|
else
|
|
l = 1;
|
|
|
|
if (l > len)
|
|
return -1;
|
|
|
|
if (!pg_utf8_islegal(s, l))
|
|
return -1;
|
|
|
|
return l;
|
|
}
|
|
|
|
/*
|
|
* The fast path of the UTF-8 verifier uses a deterministic finite automaton
|
|
* (DFA) for multibyte characters. In a traditional table-driven DFA, the
|
|
* input byte and current state are used to compute an index into an array of
|
|
* state transitions. Since the address of the next transition is dependent
|
|
* on this computation, there is latency in executing the load instruction,
|
|
* and the CPU is not kept busy.
|
|
*
|
|
* Instead, we use a "shift-based" DFA as described by Per Vognsen:
|
|
*
|
|
* https://gist.github.com/pervognsen/218ea17743e1442e59bb60d29b1aa725
|
|
*
|
|
* In a shift-based DFA, the input byte is an index into array of integers
|
|
* whose bit pattern encodes the state transitions. To compute the next
|
|
* state, we simply right-shift the integer by the current state and apply a
|
|
* mask. In this scheme, the address of the transition only depends on the
|
|
* input byte, so there is better pipelining.
|
|
*
|
|
* The naming convention for states and transitions was adopted from a UTF-8
|
|
* to UTF-16/32 transcoder, whose table is reproduced below:
|
|
*
|
|
* https://github.com/BobSteagall/utf_utils/blob/6b7a465265de2f5fa6133d653df0c9bdd73bbcf8/src/utf_utils.cpp
|
|
*
|
|
* ILL ASC CR1 CR2 CR3 L2A L3A L3B L3C L4A L4B L4C CLASS / STATE
|
|
* ==========================================================================
|
|
* err, END, err, err, err, CS1, P3A, CS2, P3B, P4A, CS3, P4B, | BGN/END
|
|
* err, err, err, err, err, err, err, err, err, err, err, err, | ERR
|
|
* |
|
|
* err, err, END, END, END, err, err, err, err, err, err, err, | CS1
|
|
* err, err, CS1, CS1, CS1, err, err, err, err, err, err, err, | CS2
|
|
* err, err, CS2, CS2, CS2, err, err, err, err, err, err, err, | CS3
|
|
* |
|
|
* err, err, err, err, CS1, err, err, err, err, err, err, err, | P3A
|
|
* err, err, CS1, CS1, err, err, err, err, err, err, err, err, | P3B
|
|
* |
|
|
* err, err, err, CS2, CS2, err, err, err, err, err, err, err, | P4A
|
|
* err, err, CS2, err, err, err, err, err, err, err, err, err, | P4B
|
|
*
|
|
* In the most straightforward implementation, a shift-based DFA for UTF-8
|
|
* requires 64-bit integers to encode the transitions, but with an SMT solver
|
|
* it's possible to find state numbers such that the transitions fit within
|
|
* 32-bit integers, as Dougall Johnson demonstrated:
|
|
*
|
|
* https://gist.github.com/dougallj/166e326de6ad4cf2c94be97a204c025f
|
|
*
|
|
* This packed representation is the reason for the seemingly odd choice of
|
|
* state values below.
|
|
*/
|
|
|
|
/* Error */
|
|
#define ERR 0
|
|
/* Begin */
|
|
#define BGN 11
|
|
/* Continuation states, expect 1/2/3 continuation bytes */
|
|
#define CS1 16
|
|
#define CS2 1
|
|
#define CS3 5
|
|
/* Partial states, where the first continuation byte has a restricted range */
|
|
#define P3A 6 /* Lead was E0, check for 3-byte overlong */
|
|
#define P3B 20 /* Lead was ED, check for surrogate */
|
|
#define P4A 25 /* Lead was F0, check for 4-byte overlong */
|
|
#define P4B 30 /* Lead was F4, check for too-large */
|
|
/* Begin and End are the same state */
|
|
#define END BGN
|
|
|
|
/* the encoded state transitions for the lookup table */
|
|
|
|
/* ASCII */
|
|
#define ASC (END << BGN)
|
|
/* 2-byte lead */
|
|
#define L2A (CS1 << BGN)
|
|
/* 3-byte lead */
|
|
#define L3A (P3A << BGN)
|
|
#define L3B (CS2 << BGN)
|
|
#define L3C (P3B << BGN)
|
|
/* 4-byte lead */
|
|
#define L4A (P4A << BGN)
|
|
#define L4B (CS3 << BGN)
|
|
#define L4C (P4B << BGN)
|
|
/* continuation byte */
|
|
#define CR1 (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4B)
|
|
#define CR2 (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3B) | (CS2 << P4A)
|
|
#define CR3 (END << CS1) | (CS1 << CS2) | (CS2 << CS3) | (CS1 << P3A) | (CS2 << P4A)
|
|
/* invalid byte */
|
|
#define ILL ERR
|
|
|
|
static const uint32 Utf8Transition[256] =
|
|
{
|
|
/* ASCII */
|
|
|
|
ILL, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,
|
|
|
|
/* continuation bytes */
|
|
|
|
/* 80..8F */
|
|
CR1, CR1, CR1, CR1, CR1, CR1, CR1, CR1,
|
|
CR1, CR1, CR1, CR1, CR1, CR1, CR1, CR1,
|
|
|
|
/* 90..9F */
|
|
CR2, CR2, CR2, CR2, CR2, CR2, CR2, CR2,
|
|
CR2, CR2, CR2, CR2, CR2, CR2, CR2, CR2,
|
|
|
|
/* A0..BF */
|
|
CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
|
|
CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
|
|
CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
|
|
CR3, CR3, CR3, CR3, CR3, CR3, CR3, CR3,
|
|
|
|
/* leading bytes */
|
|
|
|
/* C0..DF */
|
|
ILL, ILL, L2A, L2A, L2A, L2A, L2A, L2A,
|
|
L2A, L2A, L2A, L2A, L2A, L2A, L2A, L2A,
|
|
L2A, L2A, L2A, L2A, L2A, L2A, L2A, L2A,
|
|
L2A, L2A, L2A, L2A, L2A, L2A, L2A, L2A,
|
|
|
|
/* E0..EF */
|
|
L3A, L3B, L3B, L3B, L3B, L3B, L3B, L3B,
|
|
L3B, L3B, L3B, L3B, L3B, L3C, L3B, L3B,
|
|
|
|
/* F0..FF */
|
|
L4A, L4B, L4B, L4B, L4C, ILL, ILL, ILL,
|
|
ILL, ILL, ILL, ILL, ILL, ILL, ILL, ILL
|
|
};
|
|
|
|
static void
|
|
utf8_advance(const unsigned char *s, uint32 *state, int len)
|
|
{
|
|
/* Note: We deliberately don't check the state's value here. */
|
|
while (len > 0)
|
|
{
|
|
/*
|
|
* It's important that the mask value is 31: In most instruction sets,
|
|
* a shift by a 32-bit operand is understood to be a shift by its mod
|
|
* 32, so the compiler should elide the mask operation.
|
|
*/
|
|
*state = Utf8Transition[*s++] >> (*state & 31);
|
|
len--;
|
|
}
|
|
|
|
*state &= 31;
|
|
}
|
|
|
|
static int
|
|
pg_utf8_verifystr(const unsigned char *s, int len)
|
|
{
|
|
const unsigned char *start = s;
|
|
const int orig_len = len;
|
|
uint32 state = BGN;
|
|
|
|
/*
|
|
* With a stride of two vector widths, gcc will unroll the loop. Even if
|
|
* the compiler can unroll a longer loop, it's not worth it because we
|
|
* must fall back to the byte-wise algorithm if we find any non-ASCII.
|
|
*/
|
|
#define STRIDE_LENGTH (2 * sizeof(Vector8))
|
|
|
|
if (len >= STRIDE_LENGTH)
|
|
{
|
|
while (len >= STRIDE_LENGTH)
|
|
{
|
|
/*
|
|
* If the chunk is all ASCII, we can skip the full UTF-8 check,
|
|
* but we must first check for a non-END state, which means the
|
|
* previous chunk ended in the middle of a multibyte sequence.
|
|
*/
|
|
if (state != END || !is_valid_ascii(s, STRIDE_LENGTH))
|
|
utf8_advance(s, &state, STRIDE_LENGTH);
|
|
|
|
s += STRIDE_LENGTH;
|
|
len -= STRIDE_LENGTH;
|
|
}
|
|
|
|
/* The error state persists, so we only need to check for it here. */
|
|
if (state == ERR)
|
|
{
|
|
/*
|
|
* Start over from the beginning with the slow path so we can
|
|
* count the valid bytes.
|
|
*/
|
|
len = orig_len;
|
|
s = start;
|
|
}
|
|
else if (state != END)
|
|
{
|
|
/*
|
|
* The fast path exited in the middle of a multibyte sequence.
|
|
* Walk backwards to find the leading byte so that the slow path
|
|
* can resume checking from there. We must always backtrack at
|
|
* least one byte, since the current byte could be e.g. an ASCII
|
|
* byte after a 2-byte lead, which is invalid.
|
|
*/
|
|
do
|
|
{
|
|
Assert(s > start);
|
|
s--;
|
|
len++;
|
|
Assert(IS_HIGHBIT_SET(*s));
|
|
} while (pg_utf_mblen(s) <= 1);
|
|
}
|
|
}
|
|
|
|
/* check remaining bytes */
|
|
while (len > 0)
|
|
{
|
|
int l;
|
|
|
|
/* fast path for ASCII-subset characters */
|
|
if (!IS_HIGHBIT_SET(*s))
|
|
{
|
|
if (*s == '\0')
|
|
break;
|
|
l = 1;
|
|
}
|
|
else
|
|
{
|
|
l = pg_utf8_verifychar(s, len);
|
|
if (l == -1)
|
|
break;
|
|
}
|
|
s += l;
|
|
len -= l;
|
|
}
|
|
|
|
return s - start;
|
|
}
|
|
|
|
/*
|
|
* Check for validity of a single UTF-8 encoded character
|
|
*
|
|
* This directly implements the rules in RFC3629. The bizarre-looking
|
|
* restrictions on the second byte are meant to ensure that there isn't
|
|
* more than one encoding of a given Unicode character point; that is,
|
|
* you may not use a longer-than-necessary byte sequence with high order
|
|
* zero bits to represent a character that would fit in fewer bytes.
|
|
* To do otherwise is to create security hazards (eg, create an apparent
|
|
* non-ASCII character that decodes to plain ASCII).
|
|
*
|
|
* length is assumed to have been obtained by pg_utf_mblen(), and the
|
|
* caller must have checked that that many bytes are present in the buffer.
|
|
*/
|
|
bool
|
|
pg_utf8_islegal(const unsigned char *source, int length)
|
|
{
|
|
unsigned char a;
|
|
|
|
switch (length)
|
|
{
|
|
default:
|
|
/* reject lengths 5 and 6 for now */
|
|
return false;
|
|
case 4:
|
|
a = source[3];
|
|
if (a < 0x80 || a > 0xBF)
|
|
return false;
|
|
/* FALL THRU */
|
|
case 3:
|
|
a = source[2];
|
|
if (a < 0x80 || a > 0xBF)
|
|
return false;
|
|
/* FALL THRU */
|
|
case 2:
|
|
a = source[1];
|
|
switch (*source)
|
|
{
|
|
case 0xE0:
|
|
if (a < 0xA0 || a > 0xBF)
|
|
return false;
|
|
break;
|
|
case 0xED:
|
|
if (a < 0x80 || a > 0x9F)
|
|
return false;
|
|
break;
|
|
case 0xF0:
|
|
if (a < 0x90 || a > 0xBF)
|
|
return false;
|
|
break;
|
|
case 0xF4:
|
|
if (a < 0x80 || a > 0x8F)
|
|
return false;
|
|
break;
|
|
default:
|
|
if (a < 0x80 || a > 0xBF)
|
|
return false;
|
|
break;
|
|
}
|
|
/* FALL THRU */
|
|
case 1:
|
|
a = *source;
|
|
if (a >= 0x80 && a < 0xC2)
|
|
return false;
|
|
if (a > 0xF4)
|
|
return false;
|
|
break;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
|
|
/*
|
|
*-------------------------------------------------------------------
|
|
* encoding info table
|
|
*-------------------------------------------------------------------
|
|
*/
|
|
const pg_wchar_tbl pg_wchar_table[] = {
|
|
[PG_SQL_ASCII] = {pg_ascii2wchar_with_len, pg_wchar2single_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifychar, pg_ascii_verifystr, 1},
|
|
[PG_EUC_JP] = {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3},
|
|
[PG_EUC_CN] = {pg_euccn2wchar_with_len, pg_wchar2euc_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifychar, pg_euccn_verifystr, 2},
|
|
[PG_EUC_KR] = {pg_euckr2wchar_with_len, pg_wchar2euc_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifychar, pg_euckr_verifystr, 3},
|
|
[PG_EUC_TW] = {pg_euctw2wchar_with_len, pg_wchar2euc_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifychar, pg_euctw_verifystr, 4},
|
|
[PG_EUC_JIS_2004] = {pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifychar, pg_eucjp_verifystr, 3},
|
|
[PG_UTF8] = {pg_utf2wchar_with_len, pg_wchar2utf_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifychar, pg_utf8_verifystr, 4},
|
|
[PG_MULE_INTERNAL] = {pg_mule2wchar_with_len, pg_wchar2mule_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifychar, pg_mule_verifystr, 4},
|
|
[PG_LATIN1] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN2] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN3] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN4] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN9] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_LATIN10] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1256] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1258] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN866] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN874] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_KOI8R] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1251] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1252] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_ISO_8859_5] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_ISO_8859_6] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_ISO_8859_7] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_ISO_8859_8] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1250] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1253] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1254] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1255] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_WIN1257] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_KOI8U] = {pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifychar, pg_latin1_verifystr, 1},
|
|
[PG_SJIS] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2},
|
|
[PG_BIG5] = {0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifychar, pg_big5_verifystr, 2},
|
|
[PG_GBK] = {0, 0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifychar, pg_gbk_verifystr, 2},
|
|
[PG_UHC] = {0, 0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifychar, pg_uhc_verifystr, 2},
|
|
[PG_GB18030] = {0, 0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifychar, pg_gb18030_verifystr, 4},
|
|
[PG_JOHAB] = {0, 0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifychar, pg_johab_verifystr, 3},
|
|
[PG_SHIFT_JIS_2004] = {0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifychar, pg_sjis_verifystr, 2},
|
|
};
|
|
|
|
/*
|
|
* Returns the byte length of a multibyte character.
|
|
*
|
|
* Caution: when dealing with text that is not certainly valid in the
|
|
* specified encoding, the result may exceed the actual remaining
|
|
* string length. Callers that are not prepared to deal with that
|
|
* should use pg_encoding_mblen_bounded() instead.
|
|
*/
|
|
int
|
|
pg_encoding_mblen(int encoding, const char *mbstr)
|
|
{
|
|
return (PG_VALID_ENCODING(encoding) ?
|
|
pg_wchar_table[encoding].mblen((const unsigned char *) mbstr) :
|
|
pg_wchar_table[PG_SQL_ASCII].mblen((const unsigned char *) mbstr));
|
|
}
|
|
|
|
/*
|
|
* Returns the byte length of a multibyte character; but not more than
|
|
* the distance to end of string.
|
|
*/
|
|
int
|
|
pg_encoding_mblen_bounded(int encoding, const char *mbstr)
|
|
{
|
|
return strnlen(mbstr, pg_encoding_mblen(encoding, mbstr));
|
|
}
|
|
|
|
/*
|
|
* Returns the display length of a multibyte character.
|
|
*/
|
|
int
|
|
pg_encoding_dsplen(int encoding, const char *mbstr)
|
|
{
|
|
return (PG_VALID_ENCODING(encoding) ?
|
|
pg_wchar_table[encoding].dsplen((const unsigned char *) mbstr) :
|
|
pg_wchar_table[PG_SQL_ASCII].dsplen((const unsigned char *) mbstr));
|
|
}
|
|
|
|
/*
|
|
* Verify the first multibyte character of the given string.
|
|
* Return its byte length if good, -1 if bad. (See comments above for
|
|
* full details of the mbverifychar API.)
|
|
*/
|
|
int
|
|
pg_encoding_verifymbchar(int encoding, const char *mbstr, int len)
|
|
{
|
|
return (PG_VALID_ENCODING(encoding) ?
|
|
pg_wchar_table[encoding].mbverifychar((const unsigned char *) mbstr, len) :
|
|
pg_wchar_table[PG_SQL_ASCII].mbverifychar((const unsigned char *) mbstr, len));
|
|
}
|
|
|
|
/*
|
|
* Verify that a string is valid for the given encoding.
|
|
* Returns the number of input bytes (<= len) that form a valid string.
|
|
* (See comments above for full details of the mbverifystr API.)
|
|
*/
|
|
int
|
|
pg_encoding_verifymbstr(int encoding, const char *mbstr, int len)
|
|
{
|
|
return (PG_VALID_ENCODING(encoding) ?
|
|
pg_wchar_table[encoding].mbverifystr((const unsigned char *) mbstr, len) :
|
|
pg_wchar_table[PG_SQL_ASCII].mbverifystr((const unsigned char *) mbstr, len));
|
|
}
|
|
|
|
/*
|
|
* fetch maximum length of a given encoding
|
|
*/
|
|
int
|
|
pg_encoding_max_length(int encoding)
|
|
{
|
|
Assert(PG_VALID_ENCODING(encoding));
|
|
|
|
return pg_wchar_table[encoding].maxmblen;
|
|
}
|