lutf8lib.c (8364B)
1 /* 2 ** $Id: lutf8lib.c $ 3 ** Standard library for UTF-8 manipulation 4 ** See Copyright Notice in lua.h 5 */ 6 7 #define lutf8lib_c 8 #define LUA_LIB 9 10 #include "lprefix.h" 11 12 13 #include <assert.h> 14 #include <limits.h> 15 #include <stdlib.h> 16 #include <string.h> 17 18 #include "lua.h" 19 20 #include "lauxlib.h" 21 #include "lualib.h" 22 #include "llimits.h" 23 24 25 #define MAXUNICODE 0x10FFFFu 26 27 #define MAXUTF 0x7FFFFFFFu 28 29 30 #define MSGInvalid "invalid UTF-8 code" 31 32 33 #define iscont(c) (((c) & 0xC0) == 0x80) 34 #define iscontp(p) iscont(*(p)) 35 36 37 /* from strlib */ 38 /* translate a relative string position: negative means back from end */ 39 static lua_Integer u_posrelat (lua_Integer pos, size_t len) { 40 if (pos >= 0) return pos; 41 else if (0u - (size_t)pos > len) return 0; 42 else return (lua_Integer)len + pos + 1; 43 } 44 45 46 /* 47 ** Decode one UTF-8 sequence, returning NULL if byte sequence is 48 ** invalid. The array 'limits' stores the minimum value for each 49 ** sequence length, to check for overlong representations. Its first 50 ** entry forces an error for non-ascii bytes with no continuation 51 ** bytes (count == 0). 52 */ 53 static const char *utf8_decode (const char *s, l_uint32 *val, int strict) { 54 static const l_uint32 limits[] = 55 {~(l_uint32)0, 0x80, 0x800, 0x10000u, 0x200000u, 0x4000000u}; 56 unsigned int c = (unsigned char)s[0]; 57 l_uint32 res = 0; /* final result */ 58 if (c < 0x80) /* ascii? */ 59 res = c; 60 else { 61 int count = 0; /* to count number of continuation bytes */ 62 for (; c & 0x40; c <<= 1) { /* while it needs continuation bytes... */ 63 unsigned int cc = (unsigned char)s[++count]; /* read next byte */ 64 if (!iscont(cc)) /* not a continuation byte? */ 65 return NULL; /* invalid byte sequence */ 66 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ 67 } 68 res |= ((l_uint32)(c & 0x7F) << (count * 5)); /* add first byte */ 69 if (count > 5 || res > MAXUTF || res < limits[count]) 70 return NULL; /* invalid byte sequence */ 71 s += count; /* skip continuation bytes read */ 72 } 73 if (strict) { 74 /* check for invalid code points; too large or surrogates */ 75 if (res > MAXUNICODE || (0xD800u <= res && res <= 0xDFFFu)) 76 return NULL; 77 } 78 if (val) *val = res; 79 return s + 1; /* +1 to include first byte */ 80 } 81 82 83 /* 84 ** utf8len(s [, i [, j [, lax]]]) --> number of characters that 85 ** start in the range [i,j], or nil + current position if 's' is not 86 ** well formed in that interval 87 */ 88 static int utflen (lua_State *L) { 89 lua_Integer n = 0; /* counter for the number of characters */ 90 size_t len; /* string length in bytes */ 91 const char *s = luaL_checklstring(L, 1, &len); 92 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 93 lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len); 94 int lax = lua_toboolean(L, 4); 95 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2, 96 "initial position out of bounds"); 97 luaL_argcheck(L, --posj < (lua_Integer)len, 3, 98 "final position out of bounds"); 99 while (posi <= posj) { 100 const char *s1 = utf8_decode(s + posi, NULL, !lax); 101 if (s1 == NULL) { /* conversion error? */ 102 luaL_pushfail(L); /* return fail ... */ 103 lua_pushinteger(L, posi + 1); /* ... and current position */ 104 return 2; 105 } 106 posi = ct_diff2S(s1 - s); 107 n++; 108 } 109 lua_pushinteger(L, n); 110 return 1; 111 } 112 113 114 /* 115 ** codepoint(s, [i, [j [, lax]]]) -> returns codepoints for all 116 ** characters that start in the range [i,j] 117 */ 118 static int codepoint (lua_State *L) { 119 size_t len; 120 const char *s = luaL_checklstring(L, 1, &len); 121 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 122 lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len); 123 int lax = lua_toboolean(L, 4); 124 int n; 125 const char *se; 126 luaL_argcheck(L, posi >= 1, 2, "out of bounds"); 127 luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of bounds"); 128 if (posi > pose) return 0; /* empty interval; return no values */ 129 if (pose - posi >= INT_MAX) /* (lua_Integer -> int) overflow? */ 130 return luaL_error(L, "string slice too long"); 131 n = (int)(pose - posi) + 1; /* upper bound for number of returns */ 132 luaL_checkstack(L, n, "string slice too long"); 133 n = 0; /* count the number of returns */ 134 se = s + pose; /* string end */ 135 for (s += posi - 1; s < se;) { 136 l_uint32 code; 137 s = utf8_decode(s, &code, !lax); 138 if (s == NULL) 139 return luaL_error(L, MSGInvalid); 140 lua_pushinteger(L, l_castU2S(code)); 141 n++; 142 } 143 return n; 144 } 145 146 147 static void pushutfchar (lua_State *L, int arg) { 148 lua_Unsigned code = (lua_Unsigned)luaL_checkinteger(L, arg); 149 luaL_argcheck(L, code <= MAXUTF, arg, "value out of range"); 150 lua_pushfstring(L, "%U", (long)code); 151 } 152 153 154 /* 155 ** utfchar(n1, n2, ...) -> char(n1)..char(n2)... 156 */ 157 static int utfchar (lua_State *L) { 158 int n = lua_gettop(L); /* number of arguments */ 159 if (n == 1) /* optimize common case of single char */ 160 pushutfchar(L, 1); 161 else { 162 int i; 163 luaL_Buffer b; 164 luaL_buffinit(L, &b); 165 for (i = 1; i <= n; i++) { 166 pushutfchar(L, i); 167 luaL_addvalue(&b); 168 } 169 luaL_pushresult(&b); 170 } 171 return 1; 172 } 173 174 175 /* 176 ** offset(s, n, [i]) -> indices where n-th character counting from 177 ** position 'i' starts and ends; 0 means character at 'i'. 178 */ 179 static int byteoffset (lua_State *L) { 180 size_t len; 181 const char *s = luaL_checklstring(L, 1, &len); 182 lua_Integer n = luaL_checkinteger(L, 2); 183 lua_Integer posi = (n >= 0) ? 1 : cast_st2S(len) + 1; 184 posi = u_posrelat(luaL_optinteger(L, 3, posi), len); 185 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 3, 186 "position out of bounds"); 187 if (n == 0) { 188 /* find beginning of current byte sequence */ 189 while (posi > 0 && iscontp(s + posi)) posi--; 190 } 191 else { 192 if (iscontp(s + posi)) 193 return luaL_error(L, "initial position is a continuation byte"); 194 if (n < 0) { 195 while (n < 0 && posi > 0) { /* move back */ 196 do { /* find beginning of previous character */ 197 posi--; 198 } while (posi > 0 && iscontp(s + posi)); 199 n++; 200 } 201 } 202 else { 203 n--; /* do not move for 1st character */ 204 while (n > 0 && posi < (lua_Integer)len) { 205 do { /* find beginning of next character */ 206 posi++; 207 } while (iscontp(s + posi)); /* (cannot pass final '\0') */ 208 n--; 209 } 210 } 211 } 212 if (n != 0) { /* did not find given character? */ 213 luaL_pushfail(L); 214 return 1; 215 } 216 lua_pushinteger(L, posi + 1); /* initial position */ 217 if ((s[posi] & 0x80) != 0) { /* multi-byte character? */ 218 do { 219 posi++; 220 } while (iscontp(s + posi + 1)); /* skip to final byte */ 221 } 222 /* else one-byte character: final position is the initial one */ 223 lua_pushinteger(L, posi + 1); /* 'posi' now is the final position */ 224 return 2; 225 } 226 227 228 static int iter_aux (lua_State *L, int strict) { 229 size_t len; 230 const char *s = luaL_checklstring(L, 1, &len); 231 lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2); 232 if (n < len) { 233 while (iscontp(s + n)) n++; /* go to next character */ 234 } 235 if (n >= len) /* (also handles original 'n' being negative) */ 236 return 0; /* no more codepoints */ 237 else { 238 l_uint32 code; 239 const char *next = utf8_decode(s + n, &code, strict); 240 if (next == NULL || iscontp(next)) 241 return luaL_error(L, MSGInvalid); 242 lua_pushinteger(L, l_castU2S(n + 1)); 243 lua_pushinteger(L, l_castU2S(code)); 244 return 2; 245 } 246 } 247 248 249 static int iter_auxstrict (lua_State *L) { 250 return iter_aux(L, 1); 251 } 252 253 static int iter_auxlax (lua_State *L) { 254 return iter_aux(L, 0); 255 } 256 257 258 static int iter_codes (lua_State *L) { 259 int lax = lua_toboolean(L, 2); 260 const char *s = luaL_checkstring(L, 1); 261 luaL_argcheck(L, !iscontp(s), 1, MSGInvalid); 262 lua_pushcfunction(L, lax ? iter_auxlax : iter_auxstrict); 263 lua_pushvalue(L, 1); 264 lua_pushinteger(L, 0); 265 return 3; 266 } 267 268 269 /* pattern to match a single UTF-8 character */ 270 #define UTF8PATT "[\0-\x7F\xC2-\xFD][\x80-\xBF]*" 271 272 273 static const luaL_Reg funcs[] = { 274 {"offset", byteoffset}, 275 {"codepoint", codepoint}, 276 {"char", utfchar}, 277 {"len", utflen}, 278 {"codes", iter_codes}, 279 /* placeholders */ 280 {"charpattern", NULL}, 281 {NULL, NULL} 282 }; 283 284 285 LUAMOD_API int luaopen_utf8 (lua_State *L) { 286 luaL_newlib(L, funcs); 287 lua_pushlstring(L, UTF8PATT, sizeof(UTF8PATT)/sizeof(char) - 1); 288 lua_setfield(L, -2, "charpattern"); 289 return 1; 290 } 291