utf8.lua (7576B)
1 -- $Id: testes/utf8.lua $ 2 -- See Copyright Notice in file all.lua 3 4 -- UTF-8 file 5 6 print "testing UTF-8 library" 7 8 local utf8 = require'utf8' 9 10 11 local function checkerror (msg, f, ...) 12 local s, err = pcall(f, ...) 13 assert(not s and string.find(err, msg)) 14 end 15 16 17 local function len (s) 18 return #string.gsub(s, "[\x80-\xBF]", "") 19 end 20 21 22 local justone = "^" .. utf8.charpattern .. "$" 23 24 -- 't' is the list of codepoints of 's' 25 local function checksyntax (s, t) 26 -- creates a string "return '\u{t[1]}...\u{t[n]}'" 27 local ts = {"return '"} 28 for i = 1, #t do ts[i + 1] = string.format("\\u{%x}", t[i]) end 29 ts[#t + 2] = "'" 30 ts = table.concat(ts) 31 -- its execution should result in 's' 32 assert(assert(load(ts))() == s) 33 end 34 35 assert(not utf8.offset("alo", 5)) 36 assert(not utf8.offset("alo", -4)) 37 38 -- 'check' makes several tests over the validity of string 's'. 39 -- 't' is the list of codepoints of 's'. 40 local function check (s, t, nonstrict) 41 local l = utf8.len(s, 1, -1, nonstrict) 42 assert(#t == l and len(s) == l) 43 assert(utf8.char(table.unpack(t)) == s) -- 't' and 's' are equivalent 44 45 assert(utf8.offset(s, 0) == 1) 46 47 checksyntax(s, t) 48 49 -- creates new table with all codepoints of 's' 50 local t1 = {utf8.codepoint(s, 1, -1, nonstrict)} 51 assert(#t == #t1) 52 for i = 1, #t do assert(t[i] == t1[i]) end -- 't' is equal to 't1' 53 54 for i = 1, l do -- for all codepoints 55 local pi, pie = utf8.offset(s, i) -- position of i-th char 56 local pi1 = utf8.offset(s, 2, pi) -- position of next char 57 assert(pi1 == pie + 1) 58 assert(string.find(string.sub(s, pi, pi1 - 1), justone)) 59 assert(utf8.offset(s, -1, pi1) == pi) 60 assert(utf8.offset(s, i - l - 1) == pi) 61 assert(pi1 - pi == #utf8.char(utf8.codepoint(s, pi, pi, nonstrict))) 62 for j = pi, pi1 - 1 do 63 local off1, off2 = utf8.offset(s, 0, j) 64 assert(off1 == pi and off2 == pi1 - 1) 65 end 66 for j = pi + 1, pi1 - 1 do 67 assert(not utf8.len(s, j)) 68 end 69 assert(utf8.len(s, pi, pi, nonstrict) == 1) 70 assert(utf8.len(s, pi, pi1 - 1, nonstrict) == 1) 71 assert(utf8.len(s, pi, -1, nonstrict) == l - i + 1) 72 assert(utf8.len(s, pi1, -1, nonstrict) == l - i) 73 assert(utf8.len(s, 1, pi, nonstrict) == i) 74 end 75 76 local expected = 1 -- expected position of "current" character 77 for i = 1, l + 1 do 78 local p, e = utf8.offset(s, i) 79 assert(p == expected) 80 expected = e + 1 81 end 82 assert(expected - 1 == #s + 1) 83 84 local i = 0 85 for p, c in utf8.codes(s, nonstrict) do 86 i = i + 1 87 assert(c == t[i] and p == utf8.offset(s, i)) 88 assert(utf8.codepoint(s, p, p, nonstrict) == c) 89 end 90 assert(i == #t) 91 92 i = 0 93 for c in string.gmatch(s, utf8.charpattern) do 94 i = i + 1 95 assert(c == utf8.char(t[i])) 96 end 97 assert(i == #t) 98 99 for i = 1, l do 100 assert(utf8.offset(s, i) == utf8.offset(s, i - l - 1, #s + 1)) 101 end 102 103 end 104 105 106 do -- error indication in utf8.len 107 local function checklen (s, p) 108 local a, b = utf8.len(s) 109 assert(not a and b == p) 110 end 111 checklen("abc\xE3def", 4) 112 checklen("\xF4\x9F\xBF", 1) 113 checklen("\xF4\x9F\xBF\xBF", 1) 114 -- spurious continuation bytes 115 checklen("汉字\x80", #("汉字") + 1) 116 checklen("\x80hello", 1) 117 checklen("hel\x80lo", 4) 118 checklen("汉字\xBF", #("汉字") + 1) 119 checklen("\xBFhello", 1) 120 checklen("hel\xBFlo", 4) 121 end 122 123 -- errors in utf8.codes 124 do 125 local function errorcodes (s) 126 checkerror("invalid UTF%-8 code", 127 function () 128 for c in utf8.codes(s) do assert(c) end 129 end) 130 end 131 errorcodes("ab\xff") 132 errorcodes("\u{110000}") 133 errorcodes("in\x80valid") 134 errorcodes("\xbfinvalid") 135 errorcodes("αλφ\xBFα") 136 137 -- calling interation function with invalid arguments 138 local f = utf8.codes("") 139 assert(f("", 2) == nil) 140 assert(f("", -1) == nil) 141 assert(f("", math.mininteger) == nil) 142 143 end 144 145 -- error in initial position for offset 146 checkerror("position out of bounds", utf8.offset, "abc", 1, 5) 147 checkerror("position out of bounds", utf8.offset, "abc", 1, -4) 148 checkerror("position out of bounds", utf8.offset, "", 1, 2) 149 checkerror("position out of bounds", utf8.offset, "", 1, -1) 150 checkerror("continuation byte", utf8.offset, "𦧺", 1, 2) 151 checkerror("continuation byte", utf8.offset, "𦧺", 1, 2) 152 checkerror("continuation byte", utf8.offset, "\x80", 1) 153 154 -- error in indices for len 155 checkerror("out of bounds", utf8.len, "abc", 0, 2) 156 checkerror("out of bounds", utf8.len, "abc", 1, 4) 157 158 159 local s = "hello World" 160 local t = {string.byte(s, 1, -1)} 161 for i = 1, utf8.len(s) do assert(t[i] == string.byte(s, i)) end 162 check(s, t) 163 164 check("汉字/漢字", {27721, 23383, 47, 28450, 23383,}) 165 166 do 167 local s = "áéí\128" 168 local t = {utf8.codepoint(s,1,#s - 1)} 169 assert(#t == 3 and t[1] == 225 and t[2] == 233 and t[3] == 237) 170 checkerror("invalid UTF%-8 code", utf8.codepoint, s, 1, #s) 171 checkerror("out of bounds", utf8.codepoint, s, #s + 1) 172 t = {utf8.codepoint(s, 4, 3)} 173 assert(#t == 0) 174 checkerror("out of bounds", utf8.codepoint, s, -(#s + 1), 1) 175 checkerror("out of bounds", utf8.codepoint, s, 1, #s + 1) 176 -- surrogates 177 assert(utf8.codepoint("\u{D7FF}") == 0xD800 - 1) 178 assert(utf8.codepoint("\u{E000}") == 0xDFFF + 1) 179 assert(utf8.codepoint("\u{D800}", 1, 1, true) == 0xD800) 180 assert(utf8.codepoint("\u{DFFF}", 1, 1, true) == 0xDFFF) 181 assert(utf8.codepoint("\u{7FFFFFFF}", 1, 1, true) == 0x7FFFFFFF) 182 end 183 184 assert(utf8.char() == "") 185 assert(utf8.char(0, 97, 98, 99, 1) == "\0abc\1") 186 187 assert(utf8.codepoint(utf8.char(0x10FFFF)) == 0x10FFFF) 188 assert(utf8.codepoint(utf8.char(0x7FFFFFFF), 1, 1, true) == (1<<31) - 1) 189 190 checkerror("value out of range", utf8.char, 0x7FFFFFFF + 1) 191 checkerror("value out of range", utf8.char, -1) 192 193 local function invalid (s) 194 checkerror("invalid UTF%-8 code", utf8.codepoint, s) 195 assert(not utf8.len(s)) 196 end 197 198 -- UTF-8 representation for 0x11ffff (value out of valid range) 199 invalid("\xF4\x9F\xBF\xBF") 200 201 -- surrogates 202 invalid("\u{D800}") 203 invalid("\u{DFFF}") 204 205 -- overlong sequences 206 invalid("\xC0\x80") -- zero 207 invalid("\xC1\xBF") -- 0x7F (should be coded in 1 byte) 208 invalid("\xE0\x9F\xBF") -- 0x7FF (should be coded in 2 bytes) 209 invalid("\xF0\x8F\xBF\xBF") -- 0xFFFF (should be coded in 3 bytes) 210 211 212 -- invalid bytes 213 invalid("\x80") -- continuation byte 214 invalid("\xBF") -- continuation byte 215 invalid("\xFE") -- invalid byte 216 invalid("\xFF") -- invalid byte 217 218 219 -- empty string 220 check("", {}) 221 222 -- minimum and maximum values for each sequence size 223 s = "\0 \x7F\z 224 \xC2\x80 \xDF\xBF\z 225 \xE0\xA0\x80 \xEF\xBF\xBF\z 226 \xF0\x90\x80\x80 \xF4\x8F\xBF\xBF" 227 s = string.gsub(s, " ", "") 228 check(s, {0,0x7F, 0x80,0x7FF, 0x800,0xFFFF, 0x10000,0x10FFFF}) 229 230 do 231 -- original UTF-8 values 232 local s = "\u{4000000}\u{7FFFFFFF}" 233 assert(#s == 12) 234 check(s, {0x4000000, 0x7FFFFFFF}, true) 235 236 s = "\u{200000}\u{3FFFFFF}" 237 assert(#s == 10) 238 check(s, {0x200000, 0x3FFFFFF}, true) 239 240 s = "\u{10000}\u{1fffff}" 241 assert(#s == 8) 242 check(s, {0x10000, 0x1FFFFF}, true) 243 end 244 245 local x = "日本語a-4\0éó" 246 check(x, {26085, 26412, 35486, 97, 45, 52, 0, 233, 243}) 247 248 249 -- Supplementary Characters 250 check("𣲷𠜎𠱓𡁻𠵼ab𠺢", 251 {0x23CB7, 0x2070E, 0x20C53, 0x2107B, 0x20D7C, 0x61, 0x62, 0x20EA2,}) 252 253 check("𨳊𩶘𦧺𨳒𥄫𤓓\xF4\x8F\xBF\xBF", 254 {0x28CCA, 0x29D98, 0x269FA, 0x28CD2, 0x2512B, 0x244D3, 0x10ffff}) 255 256 257 local i = 0 258 for p, c in string.gmatch(x, "()(" .. utf8.charpattern .. ")") do 259 i = i + 1 260 assert(utf8.offset(x, i) == p) 261 assert(utf8.len(x, p) == utf8.len(x) - i + 1) 262 assert(utf8.len(c) == 1) 263 for j = 1, #c - 1 do 264 assert(utf8.offset(x, 0, p + j - 1) == p) 265 end 266 end 267 268 print'ok' 269