lua

A copy of the Lua development repository
Log | Files | Refs | README

commit ba7da13ec5938f978c37d63aa40a3e340b301f79
parent da37ac9c7894186a0e2e0e6f1f5f00b825fd1555
Author: Roberto Ierusalimschy <roberto@inf.puc-rio.br>
Date:   Thu, 27 Dec 2018 14:32:02 -0200

Changes in the control of C-stack overflow

  * unification of the 'nny' and 'nCcalls' counters;
  * external C functions ('lua_CFunction') count more "slots" in
    the C stack (to allow for their possible use of buffers)
  * added a new test script specific for C-stack overflows. (Most
    of those tests were already present, but concentrating them
    in a single script easies the task of checking whether
    'LUAI_MAXCCALLS' is adequate in a system.)

Diffstat:
Mlapi.c | 4++--
Mldo.c | 40++++++++++++++++------------------------
Mllimits.h | 6++++--
Mlparser.c | 8+++++---
Mlstate.c | 49+++++++++++++++++++++++++++++++++----------------
Mlstate.h | 52++++++++++++++++++++++++++++++++++++++++++----------
Mltests.h | 2+-
Mluaconf.h | 4++--
Mtestes/all.lua | 1+
Mtestes/coroutine.lua | 4++--
Atestes/cstack.lua | 62++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mtestes/pm.lua | 12------------
12 files changed, 170 insertions(+), 74 deletions(-)

diff --git a/lapi.c b/lapi.c @@ -956,7 +956,7 @@ LUA_API void lua_callk (lua_State *L, int nargs, int nresults, api_check(L, L->status == LUA_OK, "cannot do calls on non-normal thread"); checkresults(L, nargs, nresults); func = L->top - (nargs+1); - if (k != NULL && L->nny == 0) { /* need to prepare continuation? */ + if (k != NULL && yieldable(L)) { /* need to prepare continuation? */ L->ci->u.c.k = k; /* save continuation */ L->ci->u.c.ctx = ctx; /* save context */ luaD_call(L, func, nresults); /* do the call */ @@ -1004,7 +1004,7 @@ LUA_API int lua_pcallk (lua_State *L, int nargs, int nresults, int errfunc, func = savestack(L, o); } c.func = L->top - (nargs+1); /* function to be called */ - if (k == NULL || L->nny > 0) { /* no continuation or no yieldable? */ + if (k == NULL || !yieldable(L)) { /* no continuation or no yieldable? */ c.nresults = nresults; /* do a 'conventional' protected call */ status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func); } diff --git a/ldo.c b/ldo.c @@ -138,7 +138,7 @@ l_noret luaD_throw (lua_State *L, int errcode) { int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { - unsigned short oldnCcalls = L->nCcalls - L->nci; + l_uint32 oldnCcalls = L->nCcalls - L->nci; struct lua_longjmp lj; lua_assert(L->nCcalls >= L->nci); lj.status = LUA_OK; @@ -513,12 +513,17 @@ void luaD_call (lua_State *L, StkId func, int nresults) { /* -** Similar to 'luaD_call', but does not allow yields during the call +** Similar to 'luaD_call', but does not allow yields during the call. +** If there is a stack overflow, freeing all CI structures will +** force the subsequent call to invoke 'luaE_extendCI', which then +** will raise any errors. */ void luaD_callnoyield (lua_State *L, StkId func, int nResults) { - L->nny++; + incXCcalls(L); + if (getCcalls(L) >= LUAI_MAXCCALLS) /* possible stack overflow? */ + luaE_freeCI(L); luaD_call(L, func, nResults); - L->nny--; + decXCcalls(L); } @@ -530,7 +535,7 @@ static void finishCcall (lua_State *L, int status) { CallInfo *ci = L->ci; int n; /* must have a continuation and must be able to call it */ - lua_assert(ci->u.c.k != NULL && L->nny == 0); + lua_assert(ci->u.c.k != NULL && yieldable(L)); /* error status can only happen in a protected call */ lua_assert((ci->callstatus & CIST_YPCALL) || status == LUA_YIELD); if (ci->callstatus & CIST_YPCALL) { /* was inside a pcall? */ @@ -601,7 +606,6 @@ static int recover (lua_State *L, int status) { luaD_seterrorobj(L, status, oldtop); L->ci = ci; L->allowhook = getoah(ci->callstatus); /* restore original 'allowhook' */ - L->nny = 0; /* should be zero to be yieldable */ luaD_shrinkstack(L); L->errfunc = ci->u.c.old_errfunc; return 1; /* continue running the coroutine */ @@ -623,13 +627,6 @@ static int resume_error (lua_State *L, const char *msg, int narg) { /* -** "Cost" in the C stack for a coroutine invocation. -*/ -#if !defined(LUAL_COROCSTK) -#define LUAL_COROCSTK 3 -#endif - -/* ** Do the work for 'lua_resume' in protected mode. Most of the work ** depends on the status of the coroutine: initial state, suspended ** inside a hook, or regularly suspended (optionally with a continuation @@ -664,7 +661,6 @@ static void resume (lua_State *L, void *ud) { LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, int *nresults) { int status; - unsigned short oldnny = L->nny; /* save "number of non-yieldable" calls */ lua_lock(L); if (L->status == LUA_OK) { /* may be starting a coroutine */ if (L->ci != &L->base_ci) /* not in base level? */ @@ -675,11 +671,10 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, if (from == NULL) L->nCcalls = 1; else /* correct 'nCcalls' for this thread */ - L->nCcalls = from->nCcalls - from->nci + L->nci + LUAL_COROCSTK; + L->nCcalls = getCcalls(from) - from->nci + L->nci + CSTACKCF; if (L->nCcalls >= LUAI_MAXCCALLS) return resume_error(L, "C stack overflow", nargs); luai_userstateresume(L, nargs); - L->nny = 0; /* allow yields */ api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs); status = luaD_rawrunprotected(L, resume, &nargs); /* continue running after recoverable errors */ @@ -698,14 +693,13 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, } *nresults = (status == LUA_YIELD) ? L->ci->u2.nyield : cast_int(L->top - (L->ci->func + 1)); - L->nny = oldnny; /* restore 'nny' */ lua_unlock(L); return status; } LUA_API int lua_isyieldable (lua_State *L) { - return (L->nny == 0); + return yieldable(L); } @@ -715,7 +709,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx, luai_userstateyield(L, nresults); lua_lock(L); api_checknelems(L, nresults); - if (unlikely(L->nny > 0)) { + if (unlikely(!yieldable(L))) { if (L != G(L)->mainthread) luaG_runerror(L, "attempt to yield across a C-call boundary"); else @@ -741,7 +735,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx, /* ** Call the C function 'func' in protected mode, restoring basic -** thread information ('allowhook', 'nny', etc.) and in particular +** thread information ('allowhook', etc.) and in particular ** its stack level in case of errors. */ int luaD_pcall (lua_State *L, Pfunc func, void *u, @@ -749,7 +743,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u, int status; CallInfo *old_ci = L->ci; lu_byte old_allowhooks = L->allowhook; - unsigned short old_nny = L->nny; ptrdiff_t old_errfunc = L->errfunc; L->errfunc = ef; status = luaD_rawrunprotected(L, func, u); @@ -757,7 +750,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u, StkId oldtop = restorestack(L, old_top); L->ci = old_ci; L->allowhook = old_allowhooks; - L->nny = old_nny; status = luaF_close(L, oldtop, status); oldtop = restorestack(L, old_top); /* previous call may change stack */ luaD_seterrorobj(L, status, oldtop); @@ -811,7 +803,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name, const char *mode) { struct SParser p; int status; - L->nny++; /* cannot yield during parsing */ + incnny(L); /* cannot yield during parsing */ p.z = z; p.name = name; p.mode = mode; p.dyd.actvar.arr = NULL; p.dyd.actvar.size = 0; p.dyd.gt.arr = NULL; p.dyd.gt.size = 0; @@ -822,7 +814,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name, luaM_freearray(L, p.dyd.actvar.arr, p.dyd.actvar.size); luaM_freearray(L, p.dyd.gt.arr, p.dyd.gt.size); luaM_freearray(L, p.dyd.label.arr, p.dyd.label.size); - L->nny--; + decnny(L); return status; } diff --git a/llimits.h b/llimits.h @@ -184,11 +184,13 @@ typedef LUAI_UACINT l_uacInt; ** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h) */ #if LUAI_BITSINT >= 32 -typedef unsigned int Instruction; +typedef unsigned int l_uint32; #else -typedef unsigned long Instruction; +typedef unsigned long l_uint32; #endif +typedef l_uint32 Instruction; + /* diff --git a/lparser.c b/lparser.c @@ -367,10 +367,12 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) { } -#define enterlevel(ls) luaE_incCcalls((ls)->L) - +/* +** Macros to limit the maximum recursion depth while parsing +*/ +#define enterlevel(ls) luaE_enterCcall((ls)->L) -#define leavelevel(ls) ((ls)->L->nCcalls--) +#define leavelevel(ls) luaE_exitCcall((ls)->L) /* diff --git a/lstate.c b/lstate.c @@ -99,24 +99,42 @@ void luaE_setdebt (global_State *g, l_mem debt) { /* ** Increment count of "C calls" and check for overflows. In case of ** a stack overflow, check appropriate error ("regular" overflow or -** overflow while handling stack overflow). If 'nCalls' is larger than -** LUAI_MAXCCALLS (which means it is handling a "regular" overflow) but -** smaller than 9/8 of LUAI_MAXCCALLS, does not report an error (to -** allow overflow handling to work) +** overflow while handling stack overflow). +** If 'nCcalls' is larger than LUAI_MAXCCALLS but smaller than +** LUAI_MAXCCALLS + CSTACKCF (plus 2 to avoid by-one errors), it means +** it has just entered the "overflow zone", so the function raises an +** overflow error. +** If 'nCcalls' is larger than LUAI_MAXCCALLS + CSTACKCF + 2 +** (which means it is already handling an overflow) but smaller than +** 9/8 of LUAI_MAXCCALLS, does not report an error (to allow message +** handling to work). +** Otherwise, report a stack overflow while handling a stack overflow +** (probably caused by a repeating error in the message handling +** function). */ -void luaE_incCcalls (lua_State *L) { - if (++L->nCcalls >= LUAI_MAXCCALLS) { - if (L->nCcalls == LUAI_MAXCCALLS) - luaG_runerror(L, "C stack overflow"); - else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS>>3))) - luaD_throw(L, LUA_ERRERR); /* error while handing stack error */ +void luaE_enterCcall (lua_State *L) { + int ncalls = getCcalls(L); + L->nCcalls++; + if (ncalls >= LUAI_MAXCCALLS) { /* possible overflow? */ + luaE_freeCI(L); /* release unused CIs */ + ncalls = getCcalls(L); /* update call count */ + if (ncalls >= LUAI_MAXCCALLS) { /* still overflow? */ + if (ncalls <= LUAI_MAXCCALLS + CSTACKCF + 2) { + /* no error before increments; raise the error now */ + L->nCcalls += (CSTACKCF + 4); /* avoid raising it again */ + luaG_runerror(L, "C stack overflow"); + } + else if (ncalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3))) + luaD_throw(L, LUA_ERRERR); /* error while handling stack error */ + } } } CallInfo *luaE_extendCI (lua_State *L) { CallInfo *ci; - luaE_incCcalls(L); + lua_assert(L->ci->next == NULL); + luaE_enterCcall(L); ci = luaM_new(L, CallInfo); lua_assert(L->ci->next == NULL); L->ci->next = ci; @@ -135,13 +153,13 @@ void luaE_freeCI (lua_State *L) { CallInfo *ci = L->ci; CallInfo *next = ci->next; ci->next = NULL; - L->nCcalls -= L->nci; /* to subtract removed elements from 'nCcalls' */ + L->nCcalls -= L->nci; /* subtract removed elements from 'nCcalls' */ while ((ci = next) != NULL) { next = ci->next; luaM_free(L, ci); L->nci--; } - L->nCcalls += L->nci; /* to subtract removed elements from 'nCcalls' */ + L->nCcalls += L->nci; /* adjust result */ } @@ -151,7 +169,7 @@ void luaE_freeCI (lua_State *L) { void luaE_shrinkCI (lua_State *L) { CallInfo *ci = L->ci; CallInfo *next2; /* next's next */ - L->nCcalls -= L->nci; /* to subtract removed elements from 'nCcalls' */ + L->nCcalls -= L->nci; /* subtract removed elements from 'nCcalls' */ /* while there are two nexts */ while (ci->next != NULL && (next2 = ci->next->next) != NULL) { luaM_free(L, ci->next); /* free next */ @@ -160,7 +178,7 @@ void luaE_shrinkCI (lua_State *L) { next2->previous = ci; ci = next2; /* keep next's next */ } - L->nCcalls += L->nci; /* to subtract removed elements from 'nCcalls' */ + L->nCcalls += L->nci; /* adjust result */ } @@ -250,7 +268,6 @@ static void preinit_thread (lua_State *L, global_State *g) { L->allowhook = 1; resethookcount(L); L->openupval = NULL; - L->nny = 1; L->status = LUA_OK; L->errfunc = 0; } diff --git a/lstate.h b/lstate.h @@ -15,7 +15,6 @@ /* - ** Some notes about garbage-collected objects: All objects in Lua must ** be kept somehow accessible until being freed, so all objects always ** belong to one (and only one) of these lists, using field 'next' of @@ -43,26 +42,58 @@ ** 'weak': tables with weak values to be cleared; ** 'ephemeron': ephemeron tables with white->white entries; ** 'allweak': tables with weak keys and/or weak values to be cleared. - */ -/* +/* ** About 'nCcalls': each thread in Lua (a lua_State) keeps a count of -** how many "C calls" it has in the C stack, to avoid C-stack overflow. +** how many "C calls" it can do in the C stack, to avoid C-stack overflow. ** This count is very rough approximation; it considers only recursive ** functions inside the interpreter, as non-recursive calls can be ** considered using a fixed (although unknown) amount of stack space. ** +** The count itself has two parts: the lower part is the count itself; +** the higher part counts the number of non-yieldable calls in the stack. +** +** Because calls to external C functions can use of unkown amount +** of space (e.g., functions using an auxiliary buffer), calls +** to these functions add more than one to the count. +** ** The proper count also includes the number of CallInfo structures ** allocated by Lua, as a kind of "potential" calls. So, when Lua ** calls a function (and "consumes" one CallInfo), it needs neither to ** increment nor to check 'nCcalls', as its use of C stack is already ** accounted for. - */ +/* number of "C stack slots" used by an external C function */ +#define CSTACKCF 10 + +/* true if this thread does not have non-yieldable calls in the stack */ +#define yieldable(L) (((L)->nCcalls & 0xffff0000) == 0) + +/* real number of C calls */ +#define getCcalls(L) ((L)->nCcalls & 0xffff) + + +/* Increment the number of non-yieldable calls */ +#define incnny(L) ((L)->nCcalls += 0x10000) + +/* Decrement the number of non-yieldable calls */ +#define decnny(L) ((L)->nCcalls -= 0x10000) + +/* Increment the number of non-yieldable calls and nCcalls */ +#define incXCcalls(L) ((L)->nCcalls += 0x10000 + CSTACKCF) + +/* Decrement the number of non-yieldable calls and nCcalls */ +#define decXCcalls(L) ((L)->nCcalls -= 0x10000 + CSTACKCF) + + + + + + struct lua_longjmp; /* defined in ldo.c */ @@ -208,8 +239,9 @@ typedef struct global_State { */ struct lua_State { CommonHeader; - unsigned short nci; /* number of items in 'ci' list */ lu_byte status; + lu_byte allowhook; + unsigned short nci; /* number of items in 'ci' list */ StkId top; /* first free slot in the stack */ global_State *l_G; CallInfo *ci; /* call info for current function */ @@ -223,13 +255,11 @@ struct lua_State { CallInfo base_ci; /* CallInfo for first level (C calling Lua) */ volatile lua_Hook hook; ptrdiff_t errfunc; /* current error handling function (stack index) */ + l_uint32 nCcalls; /* number of allowed nested C calls - 'nci' */ int stacksize; int basehookcount; int hookcount; - unsigned short nny; /* number of non-yieldable calls in stack */ - unsigned short nCcalls; /* number of nested C calls + 'nny' */ l_signalT hookmask; - lu_byte allowhook; }; @@ -283,8 +313,10 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1); LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L); LUAI_FUNC void luaE_freeCI (lua_State *L); LUAI_FUNC void luaE_shrinkCI (lua_State *L); -LUAI_FUNC void luaE_incCcalls (lua_State *L); +LUAI_FUNC void luaE_enterCcall (lua_State *L); + +#define luaE_exitCcall(L) ((L)->nCcalls--) #endif diff --git a/ltests.h b/ltests.h @@ -31,7 +31,7 @@ /* compiled with -O0, Lua uses a lot of C stack space... */ #undef LUAI_MAXCCALLS -#define LUAI_MAXCCALLS 200 +#define LUAI_MAXCCALLS 400 /* to avoid warnings, and to make sure value is really unused */ #define UNUSED(x) (x=0, (void)(x)) diff --git a/luaconf.h b/luaconf.h @@ -695,14 +695,14 @@ /* @@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system. ** CHANGE it if it uses too much C-stack space. (For long double, -** 'string.format("%.99f", -1e4932)' needs 5034 bytes, so a +** 'string.format("%.99f", -1e4932)' needs 5052 bytes, so a ** smaller buffer would force a memory allocation for each call to ** 'string.format'.) */ #if LUA_FLOAT_TYPE == LUA_FLOAT_LONGDOUBLE #define LUAL_BUFFERSIZE 8192 #else -#define LUAL_BUFFERSIZE ((int)(0x80 * sizeof(void*) * sizeof(lua_Integer))) +#define LUAL_BUFFERSIZE ((int)(16 * sizeof(void*) * sizeof(lua_Number))) #endif /* diff --git a/testes/all.lua b/testes/all.lua @@ -172,6 +172,7 @@ if not _G._soft then assert(f() == 'b') assert(f() == 'a') end +dofile('cstack.lua') dofile('nextvar.lua') dofile('pm.lua') dofile('utf8.lua') diff --git a/testes/coroutine.lua b/testes/coroutine.lua @@ -107,7 +107,7 @@ function filter (p, g) end) end -local x = gen(100) +local x = gen(80) local a = {} while 1 do local n = x() @@ -116,7 +116,7 @@ while 1 do x = filter(n, x) end -assert(#a == 25 and a[#a] == 97) +assert(#a == 22 and a[#a] == 79) x, a = nil diff --git a/testes/cstack.lua b/testes/cstack.lua @@ -0,0 +1,62 @@ +-- $Id: testes/cstack.lua $ +-- See Copyright Notice in file all.lua + +print"testing C-stack overflow detection" + +-- Segmentation faults in these tests probably result from a C-stack +-- overflow. To avoid these errors, recompile Lua with a smaller +-- value for the constant 'LUAI_MAXCCALLS' or else ensure a larger +-- stack for the program. + +local function checkerror (msg, f, ...) + local s, err = pcall(f, ...) + assert(not s and string.find(err, msg)) +end + + +do -- simple recursion + local count = 0 + local function foo () + count = count + 1 + foo() + end + checkerror("stack overflow", foo) + print(" maximum recursion: " .. count) +end + + +-- bug since 2.5 (C-stack overflow in recursion inside pattern matching) +do + local function f (size) + local s = string.rep("a", size) + local p = string.rep(".?", size) + return string.match(s, p) + end + local m = f(80) + assert(#m == 80) + checkerror("too complex", f, 200000) +end + + +-- testing stack-overflow in recursive 'gsub' +do + local count = 0 + local function foo () + count = count + 1 + string.gsub("a", ".", foo) + end + checkerror("stack overflow", foo) + print(" maximum 'gsub' nest (calls): " .. count) + + -- can be done with metamethods, too + count = 0 + local t = setmetatable({}, {__index = foo}) + foo = function () + count = count + 1 + string.gsub("a", ".", t) + end + checkerror("stack overflow", foo) + print(" maximum 'gsub' nest (metamethods): " .. count) +end + +print'OK' diff --git a/testes/pm.lua b/testes/pm.lua @@ -237,18 +237,6 @@ checkerror("invalid capture index %%0", string.gsub, "alo", "(%0)", "a") checkerror("invalid capture index %%1", string.gsub, "alo", "(%1)", "a") checkerror("invalid use of '%%'", string.gsub, "alo", ".", "%x") --- bug since 2.5 (C-stack overflow) -do - local function f (size) - local s = string.rep("a", size) - local p = string.rep(".?", size) - return pcall(string.match, s, p) - end - local r, m = f(80) - assert(r and #m == 80) - r, m = f(200000) - assert(not r and string.find(m, "too complex")) -end if not _soft then print("big strings")