lua

A copy of the Lua development repository
Log | Files | Refs | README

lutf8lib.c (8364B)


      1 /*
      2 ** $Id: lutf8lib.c $
      3 ** Standard library for UTF-8 manipulation
      4 ** See Copyright Notice in lua.h
      5 */
      6 
      7 #define lutf8lib_c
      8 #define LUA_LIB
      9 
     10 #include "lprefix.h"
     11 
     12 
     13 #include <assert.h>
     14 #include <limits.h>
     15 #include <stdlib.h>
     16 #include <string.h>
     17 
     18 #include "lua.h"
     19 
     20 #include "lauxlib.h"
     21 #include "lualib.h"
     22 #include "llimits.h"
     23 
     24 
     25 #define MAXUNICODE	0x10FFFFu
     26 
     27 #define MAXUTF		0x7FFFFFFFu
     28 
     29 
     30 #define MSGInvalid	"invalid UTF-8 code"
     31 
     32 
     33 #define iscont(c)	(((c) & 0xC0) == 0x80)
     34 #define iscontp(p)	iscont(*(p))
     35 
     36 
     37 /* from strlib */
     38 /* translate a relative string position: negative means back from end */
     39 static lua_Integer u_posrelat (lua_Integer pos, size_t len) {
     40   if (pos >= 0) return pos;
     41   else if (0u - (size_t)pos > len) return 0;
     42   else return (lua_Integer)len + pos + 1;
     43 }
     44 
     45 
     46 /*
     47 ** Decode one UTF-8 sequence, returning NULL if byte sequence is
     48 ** invalid.  The array 'limits' stores the minimum value for each
     49 ** sequence length, to check for overlong representations. Its first
     50 ** entry forces an error for non-ascii bytes with no continuation
     51 ** bytes (count == 0).
     52 */
     53 static const char *utf8_decode (const char *s, l_uint32 *val, int strict) {
     54   static const l_uint32 limits[] =
     55         {~(l_uint32)0, 0x80, 0x800, 0x10000u, 0x200000u, 0x4000000u};
     56   unsigned int c = (unsigned char)s[0];
     57   l_uint32 res = 0;  /* final result */
     58   if (c < 0x80)  /* ascii? */
     59     res = c;
     60   else {
     61     int count = 0;  /* to count number of continuation bytes */
     62     for (; c & 0x40; c <<= 1) {  /* while it needs continuation bytes... */
     63       unsigned int cc = (unsigned char)s[++count];  /* read next byte */
     64       if (!iscont(cc))  /* not a continuation byte? */
     65         return NULL;  /* invalid byte sequence */
     66       res = (res << 6) | (cc & 0x3F);  /* add lower 6 bits from cont. byte */
     67     }
     68     res |= ((l_uint32)(c & 0x7F) << (count * 5));  /* add first byte */
     69     if (count > 5 || res > MAXUTF || res < limits[count])
     70       return NULL;  /* invalid byte sequence */
     71     s += count;  /* skip continuation bytes read */
     72   }
     73   if (strict) {
     74     /* check for invalid code points; too large or surrogates */
     75     if (res > MAXUNICODE || (0xD800u <= res && res <= 0xDFFFu))
     76       return NULL;
     77   }
     78   if (val) *val = res;
     79   return s + 1;  /* +1 to include first byte */
     80 }
     81 
     82 
     83 /*
     84 ** utf8len(s [, i [, j [, lax]]]) --> number of characters that
     85 ** start in the range [i,j], or nil + current position if 's' is not
     86 ** well formed in that interval
     87 */
     88 static int utflen (lua_State *L) {
     89   lua_Integer n = 0;  /* counter for the number of characters */
     90   size_t len;  /* string length in bytes */
     91   const char *s = luaL_checklstring(L, 1, &len);
     92   lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
     93   lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len);
     94   int lax = lua_toboolean(L, 4);
     95   luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2,
     96                    "initial position out of bounds");
     97   luaL_argcheck(L, --posj < (lua_Integer)len, 3,
     98                    "final position out of bounds");
     99   while (posi <= posj) {
    100     const char *s1 = utf8_decode(s + posi, NULL, !lax);
    101     if (s1 == NULL) {  /* conversion error? */
    102       luaL_pushfail(L);  /* return fail ... */
    103       lua_pushinteger(L, posi + 1);  /* ... and current position */
    104       return 2;
    105     }
    106     posi = ct_diff2S(s1 - s);
    107     n++;
    108   }
    109   lua_pushinteger(L, n);
    110   return 1;
    111 }
    112 
    113 
    114 /*
    115 ** codepoint(s, [i, [j [, lax]]]) -> returns codepoints for all
    116 ** characters that start in the range [i,j]
    117 */
    118 static int codepoint (lua_State *L) {
    119   size_t len;
    120   const char *s = luaL_checklstring(L, 1, &len);
    121   lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len);
    122   lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len);
    123   int lax = lua_toboolean(L, 4);
    124   int n;
    125   const char *se;
    126   luaL_argcheck(L, posi >= 1, 2, "out of bounds");
    127   luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of bounds");
    128   if (posi > pose) return 0;  /* empty interval; return no values */
    129   if (pose - posi >= INT_MAX)  /* (lua_Integer -> int) overflow? */
    130     return luaL_error(L, "string slice too long");
    131   n = (int)(pose -  posi) + 1;  /* upper bound for number of returns */
    132   luaL_checkstack(L, n, "string slice too long");
    133   n = 0;  /* count the number of returns */
    134   se = s + pose;  /* string end */
    135   for (s += posi - 1; s < se;) {
    136     l_uint32 code;
    137     s = utf8_decode(s, &code, !lax);
    138     if (s == NULL)
    139       return luaL_error(L, MSGInvalid);
    140     lua_pushinteger(L, l_castU2S(code));
    141     n++;
    142   }
    143   return n;
    144 }
    145 
    146 
    147 static void pushutfchar (lua_State *L, int arg) {
    148   lua_Unsigned code = (lua_Unsigned)luaL_checkinteger(L, arg);
    149   luaL_argcheck(L, code <= MAXUTF, arg, "value out of range");
    150   lua_pushfstring(L, "%U", (long)code);
    151 }
    152 
    153 
    154 /*
    155 ** utfchar(n1, n2, ...)  -> char(n1)..char(n2)...
    156 */
    157 static int utfchar (lua_State *L) {
    158   int n = lua_gettop(L);  /* number of arguments */
    159   if (n == 1)  /* optimize common case of single char */
    160     pushutfchar(L, 1);
    161   else {
    162     int i;
    163     luaL_Buffer b;
    164     luaL_buffinit(L, &b);
    165     for (i = 1; i <= n; i++) {
    166       pushutfchar(L, i);
    167       luaL_addvalue(&b);
    168     }
    169     luaL_pushresult(&b);
    170   }
    171   return 1;
    172 }
    173 
    174 
    175 /*
    176 ** offset(s, n, [i])  -> indices where n-th character counting from
    177 **   position 'i' starts and ends; 0 means character at 'i'.
    178 */
    179 static int byteoffset (lua_State *L) {
    180   size_t len;
    181   const char *s = luaL_checklstring(L, 1, &len);
    182   lua_Integer n  = luaL_checkinteger(L, 2);
    183   lua_Integer posi = (n >= 0) ? 1 : cast_st2S(len) + 1;
    184   posi = u_posrelat(luaL_optinteger(L, 3, posi), len);
    185   luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 3,
    186                    "position out of bounds");
    187   if (n == 0) {
    188     /* find beginning of current byte sequence */
    189     while (posi > 0 && iscontp(s + posi)) posi--;
    190   }
    191   else {
    192     if (iscontp(s + posi))
    193       return luaL_error(L, "initial position is a continuation byte");
    194     if (n < 0) {
    195       while (n < 0 && posi > 0) {  /* move back */
    196         do {  /* find beginning of previous character */
    197           posi--;
    198         } while (posi > 0 && iscontp(s + posi));
    199         n++;
    200       }
    201     }
    202     else {
    203       n--;  /* do not move for 1st character */
    204       while (n > 0 && posi < (lua_Integer)len) {
    205         do {  /* find beginning of next character */
    206           posi++;
    207         } while (iscontp(s + posi));  /* (cannot pass final '\0') */
    208         n--;
    209       }
    210     }
    211   }
    212   if (n != 0) {  /* did not find given character? */
    213     luaL_pushfail(L);
    214     return 1;
    215   }
    216   lua_pushinteger(L, posi + 1);  /* initial position */
    217   if ((s[posi] & 0x80) != 0) {  /* multi-byte character? */
    218     do {
    219       posi++;
    220     } while (iscontp(s + posi + 1));  /* skip to final byte */
    221   }
    222   /* else one-byte character: final position is the initial one */
    223   lua_pushinteger(L, posi + 1);  /* 'posi' now is the final position */
    224   return 2;
    225 }
    226 
    227 
    228 static int iter_aux (lua_State *L, int strict) {
    229   size_t len;
    230   const char *s = luaL_checklstring(L, 1, &len);
    231   lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2);
    232   if (n < len) {
    233     while (iscontp(s + n)) n++;  /* go to next character */
    234   }
    235   if (n >= len)  /* (also handles original 'n' being negative) */
    236     return 0;  /* no more codepoints */
    237   else {
    238     l_uint32 code;
    239     const char *next = utf8_decode(s + n, &code, strict);
    240     if (next == NULL || iscontp(next))
    241       return luaL_error(L, MSGInvalid);
    242     lua_pushinteger(L, l_castU2S(n + 1));
    243     lua_pushinteger(L, l_castU2S(code));
    244     return 2;
    245   }
    246 }
    247 
    248 
    249 static int iter_auxstrict (lua_State *L) {
    250   return iter_aux(L, 1);
    251 }
    252 
    253 static int iter_auxlax (lua_State *L) {
    254   return iter_aux(L, 0);
    255 }
    256 
    257 
    258 static int iter_codes (lua_State *L) {
    259   int lax = lua_toboolean(L, 2);
    260   const char *s = luaL_checkstring(L, 1);
    261   luaL_argcheck(L, !iscontp(s), 1, MSGInvalid);
    262   lua_pushcfunction(L, lax ? iter_auxlax : iter_auxstrict);
    263   lua_pushvalue(L, 1);
    264   lua_pushinteger(L, 0);
    265   return 3;
    266 }
    267 
    268 
    269 /* pattern to match a single UTF-8 character */
    270 #define UTF8PATT	"[\0-\x7F\xC2-\xFD][\x80-\xBF]*"
    271 
    272 
    273 static const luaL_Reg funcs[] = {
    274   {"offset", byteoffset},
    275   {"codepoint", codepoint},
    276   {"char", utfchar},
    277   {"len", utflen},
    278   {"codes", iter_codes},
    279   /* placeholders */
    280   {"charpattern", NULL},
    281   {NULL, NULL}
    282 };
    283 
    284 
    285 LUAMOD_API int luaopen_utf8 (lua_State *L) {
    286   luaL_newlib(L, funcs);
    287   lua_pushlstring(L, UTF8PATT, sizeof(UTF8PATT)/sizeof(char) - 1);
    288   lua_setfield(L, -2, "charpattern");
    289   return 1;
    290 }
    291