lua

A copy of the Lua development repository
Log | Files | Refs | README

utf8.lua (7576B)


      1 -- $Id: testes/utf8.lua $
      2 -- See Copyright Notice in file all.lua
      3 
      4 -- UTF-8 file
      5 
      6 print "testing UTF-8 library"
      7 
      8 local utf8 = require'utf8'
      9 
     10 
     11 local function checkerror (msg, f, ...)
     12   local s, err = pcall(f, ...)
     13   assert(not s and string.find(err, msg))
     14 end
     15 
     16 
     17 local function len (s)
     18   return #string.gsub(s, "[\x80-\xBF]", "")
     19 end
     20 
     21 
     22 local justone = "^" .. utf8.charpattern .. "$"
     23 
     24 -- 't' is the list of codepoints of 's'
     25 local function checksyntax (s, t)
     26   -- creates a string "return '\u{t[1]}...\u{t[n]}'"
     27   local ts = {"return '"}
     28   for i = 1, #t do ts[i + 1] = string.format("\\u{%x}", t[i]) end
     29   ts[#t + 2] = "'"
     30   ts = table.concat(ts)
     31   -- its execution should result in 's'
     32   assert(assert(load(ts))() == s)
     33 end
     34 
     35 assert(not utf8.offset("alo", 5))
     36 assert(not utf8.offset("alo", -4))
     37 
     38 -- 'check' makes several tests over the validity of string 's'.
     39 -- 't' is the list of codepoints of 's'.
     40 local function check (s, t, nonstrict)
     41   local l = utf8.len(s, 1, -1, nonstrict)
     42   assert(#t == l and len(s) == l)
     43   assert(utf8.char(table.unpack(t)) == s)   -- 't' and 's' are equivalent
     44 
     45   assert(utf8.offset(s, 0) == 1)
     46 
     47   checksyntax(s, t)
     48 
     49   -- creates new table with all codepoints of 's'
     50   local t1 = {utf8.codepoint(s, 1, -1, nonstrict)}
     51   assert(#t == #t1)
     52   for i = 1, #t do assert(t[i] == t1[i]) end   -- 't' is equal to 't1'
     53 
     54   for i = 1, l do   -- for all codepoints
     55     local pi, pie = utf8.offset(s, i)        -- position of i-th char
     56     local pi1 = utf8.offset(s, 2, pi)   -- position of next char
     57     assert(pi1 == pie + 1)
     58     assert(string.find(string.sub(s, pi, pi1 - 1), justone))
     59     assert(utf8.offset(s, -1, pi1) == pi)
     60     assert(utf8.offset(s, i - l - 1) == pi)
     61     assert(pi1 - pi == #utf8.char(utf8.codepoint(s, pi, pi, nonstrict)))
     62     for j = pi, pi1 - 1 do
     63       local off1, off2 = utf8.offset(s, 0, j)
     64       assert(off1 == pi and off2 == pi1 - 1)
     65     end
     66     for j = pi + 1, pi1 - 1 do
     67       assert(not utf8.len(s, j))
     68     end
     69     assert(utf8.len(s, pi, pi, nonstrict) == 1)
     70     assert(utf8.len(s, pi, pi1 - 1, nonstrict) == 1)
     71     assert(utf8.len(s, pi, -1, nonstrict) == l - i + 1)
     72     assert(utf8.len(s, pi1, -1, nonstrict) == l - i)
     73     assert(utf8.len(s, 1, pi, nonstrict) == i)
     74   end
     75 
     76   local expected = 1    -- expected position of "current" character
     77   for i = 1, l + 1 do
     78     local p, e = utf8.offset(s, i)
     79     assert(p == expected)
     80     expected = e + 1
     81   end
     82   assert(expected - 1 == #s + 1)
     83 
     84   local i = 0
     85   for p, c in utf8.codes(s, nonstrict) do
     86     i = i + 1
     87     assert(c == t[i] and p == utf8.offset(s, i))
     88     assert(utf8.codepoint(s, p, p, nonstrict) == c)
     89   end
     90   assert(i == #t)
     91 
     92   i = 0
     93   for c in string.gmatch(s, utf8.charpattern) do
     94     i = i + 1
     95     assert(c == utf8.char(t[i]))
     96   end
     97   assert(i == #t)
     98 
     99   for i = 1, l do
    100     assert(utf8.offset(s, i) == utf8.offset(s, i - l - 1, #s + 1))
    101   end
    102 
    103 end
    104 
    105 
    106 do    -- error indication in utf8.len
    107   local function checklen (s, p)
    108     local a, b = utf8.len(s)
    109     assert(not a and b == p)
    110   end
    111   checklen("abc\xE3def", 4)
    112   checklen("\xF4\x9F\xBF", 1)
    113   checklen("\xF4\x9F\xBF\xBF", 1)
    114   -- spurious continuation bytes
    115   checklen("汉字\x80", #("汉字") + 1)
    116   checklen("\x80hello", 1)
    117   checklen("hel\x80lo", 4)
    118   checklen("汉字\xBF", #("汉字") + 1)
    119   checklen("\xBFhello", 1)
    120   checklen("hel\xBFlo", 4)
    121 end
    122 
    123 -- errors in utf8.codes
    124 do
    125   local function errorcodes (s)
    126     checkerror("invalid UTF%-8 code",
    127       function ()
    128         for c in utf8.codes(s) do assert(c) end
    129       end)
    130   end
    131   errorcodes("ab\xff")
    132   errorcodes("\u{110000}")
    133   errorcodes("in\x80valid")
    134   errorcodes("\xbfinvalid")
    135   errorcodes("αλφ\xBFα")
    136 
    137   -- calling interation function with invalid arguments
    138   local f = utf8.codes("")
    139   assert(f("", 2) == nil)
    140   assert(f("", -1) == nil)
    141   assert(f("", math.mininteger) == nil)
    142 
    143 end
    144 
    145 -- error in initial position for offset
    146 checkerror("position out of bounds", utf8.offset, "abc", 1, 5)
    147 checkerror("position out of bounds", utf8.offset, "abc", 1, -4)
    148 checkerror("position out of bounds", utf8.offset, "", 1, 2)
    149 checkerror("position out of bounds", utf8.offset, "", 1, -1)
    150 checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
    151 checkerror("continuation byte", utf8.offset, "𦧺", 1, 2)
    152 checkerror("continuation byte", utf8.offset, "\x80", 1)
    153 
    154 -- error in indices for len
    155 checkerror("out of bounds", utf8.len, "abc", 0, 2)
    156 checkerror("out of bounds", utf8.len, "abc", 1, 4)
    157 
    158 
    159 local s = "hello World"
    160 local t = {string.byte(s, 1, -1)}
    161 for i = 1, utf8.len(s) do assert(t[i] == string.byte(s, i)) end
    162 check(s, t)
    163 
    164 check("汉字/漢字", {27721, 23383, 47, 28450, 23383,})
    165 
    166 do
    167   local s = "áéí\128"
    168   local t = {utf8.codepoint(s,1,#s - 1)}
    169   assert(#t == 3 and t[1] == 225 and t[2] == 233 and t[3] == 237)
    170   checkerror("invalid UTF%-8 code", utf8.codepoint, s, 1, #s)
    171   checkerror("out of bounds", utf8.codepoint, s, #s + 1)
    172   t = {utf8.codepoint(s, 4, 3)}
    173   assert(#t == 0)
    174   checkerror("out of bounds", utf8.codepoint, s, -(#s + 1), 1)
    175   checkerror("out of bounds", utf8.codepoint, s, 1, #s + 1)
    176   -- surrogates
    177   assert(utf8.codepoint("\u{D7FF}") == 0xD800 - 1)
    178   assert(utf8.codepoint("\u{E000}") == 0xDFFF + 1)
    179   assert(utf8.codepoint("\u{D800}", 1, 1, true) == 0xD800)
    180   assert(utf8.codepoint("\u{DFFF}", 1, 1, true) == 0xDFFF)
    181   assert(utf8.codepoint("\u{7FFFFFFF}", 1, 1, true) == 0x7FFFFFFF)
    182 end
    183 
    184 assert(utf8.char() == "")
    185 assert(utf8.char(0, 97, 98, 99, 1) == "\0abc\1")
    186 
    187 assert(utf8.codepoint(utf8.char(0x10FFFF)) == 0x10FFFF)
    188 assert(utf8.codepoint(utf8.char(0x7FFFFFFF), 1, 1, true) == (1<<31) - 1)
    189 
    190 checkerror("value out of range", utf8.char, 0x7FFFFFFF + 1)
    191 checkerror("value out of range", utf8.char, -1)
    192 
    193 local function invalid (s)
    194   checkerror("invalid UTF%-8 code", utf8.codepoint, s)
    195   assert(not utf8.len(s))
    196 end
    197 
    198 -- UTF-8 representation for 0x11ffff (value out of valid range)
    199 invalid("\xF4\x9F\xBF\xBF")
    200 
    201 -- surrogates
    202 invalid("\u{D800}")
    203 invalid("\u{DFFF}")
    204 
    205 -- overlong sequences
    206 invalid("\xC0\x80")          -- zero
    207 invalid("\xC1\xBF")          -- 0x7F (should be coded in 1 byte)
    208 invalid("\xE0\x9F\xBF")      -- 0x7FF (should be coded in 2 bytes)
    209 invalid("\xF0\x8F\xBF\xBF")  -- 0xFFFF (should be coded in 3 bytes)
    210 
    211 
    212 -- invalid bytes
    213 invalid("\x80")  -- continuation byte
    214 invalid("\xBF")  -- continuation byte
    215 invalid("\xFE")  -- invalid byte
    216 invalid("\xFF")  -- invalid byte
    217 
    218 
    219 -- empty string
    220 check("", {})
    221 
    222 -- minimum and maximum values for each sequence size
    223 s = "\0 \x7F\z
    224      \xC2\x80 \xDF\xBF\z
    225      \xE0\xA0\x80 \xEF\xBF\xBF\z
    226      \xF0\x90\x80\x80  \xF4\x8F\xBF\xBF"
    227 s = string.gsub(s, " ", "")
    228 check(s, {0,0x7F, 0x80,0x7FF, 0x800,0xFFFF, 0x10000,0x10FFFF})
    229 
    230 do
    231   -- original UTF-8 values
    232   local s = "\u{4000000}\u{7FFFFFFF}"
    233   assert(#s == 12)
    234   check(s, {0x4000000, 0x7FFFFFFF}, true)
    235 
    236   s = "\u{200000}\u{3FFFFFF}"
    237   assert(#s == 10)
    238   check(s, {0x200000, 0x3FFFFFF}, true)
    239 
    240   s = "\u{10000}\u{1fffff}"
    241   assert(#s == 8)
    242   check(s, {0x10000, 0x1FFFFF}, true)
    243 end
    244 
    245 local x = "日本語a-4\0éó"
    246 check(x, {26085, 26412, 35486, 97, 45, 52, 0, 233, 243})
    247 
    248 
    249 -- Supplementary Characters
    250 check("𣲷𠜎𠱓𡁻𠵼ab𠺢",
    251       {0x23CB7, 0x2070E, 0x20C53, 0x2107B, 0x20D7C, 0x61, 0x62, 0x20EA2,})
    252 
    253 check("𨳊𩶘𦧺𨳒𥄫𤓓\xF4\x8F\xBF\xBF",
    254       {0x28CCA, 0x29D98, 0x269FA, 0x28CD2, 0x2512B, 0x244D3, 0x10ffff})
    255 
    256 
    257 local i = 0
    258 for p, c in string.gmatch(x, "()(" .. utf8.charpattern .. ")") do
    259   i = i + 1
    260   assert(utf8.offset(x, i) == p)
    261   assert(utf8.len(x, p) == utf8.len(x) - i + 1)
    262   assert(utf8.len(c) == 1)
    263   for j = 1, #c - 1 do
    264     assert(utf8.offset(x, 0, p + j - 1) == p)
    265   end
    266 end
    267 
    268 print'ok'
    269