commit ba7da13ec5938f978c37d63aa40a3e340b301f79
parent da37ac9c7894186a0e2e0e6f1f5f00b825fd1555
Author: Roberto Ierusalimschy <roberto@inf.puc-rio.br>
Date: Thu, 27 Dec 2018 14:32:02 -0200
Changes in the control of C-stack overflow
* unification of the 'nny' and 'nCcalls' counters;
* external C functions ('lua_CFunction') count more "slots" in
the C stack (to allow for their possible use of buffers)
* added a new test script specific for C-stack overflows. (Most
of those tests were already present, but concentrating them
in a single script easies the task of checking whether
'LUAI_MAXCCALLS' is adequate in a system.)
Diffstat:
12 files changed, 170 insertions(+), 74 deletions(-)
diff --git a/lapi.c b/lapi.c
@@ -956,7 +956,7 @@ LUA_API void lua_callk (lua_State *L, int nargs, int nresults,
api_check(L, L->status == LUA_OK, "cannot do calls on non-normal thread");
checkresults(L, nargs, nresults);
func = L->top - (nargs+1);
- if (k != NULL && L->nny == 0) { /* need to prepare continuation? */
+ if (k != NULL && yieldable(L)) { /* need to prepare continuation? */
L->ci->u.c.k = k; /* save continuation */
L->ci->u.c.ctx = ctx; /* save context */
luaD_call(L, func, nresults); /* do the call */
@@ -1004,7 +1004,7 @@ LUA_API int lua_pcallk (lua_State *L, int nargs, int nresults, int errfunc,
func = savestack(L, o);
}
c.func = L->top - (nargs+1); /* function to be called */
- if (k == NULL || L->nny > 0) { /* no continuation or no yieldable? */
+ if (k == NULL || !yieldable(L)) { /* no continuation or no yieldable? */
c.nresults = nresults; /* do a 'conventional' protected call */
status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func);
}
diff --git a/ldo.c b/ldo.c
@@ -138,7 +138,7 @@ l_noret luaD_throw (lua_State *L, int errcode) {
int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) {
- unsigned short oldnCcalls = L->nCcalls - L->nci;
+ l_uint32 oldnCcalls = L->nCcalls - L->nci;
struct lua_longjmp lj;
lua_assert(L->nCcalls >= L->nci);
lj.status = LUA_OK;
@@ -513,12 +513,17 @@ void luaD_call (lua_State *L, StkId func, int nresults) {
/*
-** Similar to 'luaD_call', but does not allow yields during the call
+** Similar to 'luaD_call', but does not allow yields during the call.
+** If there is a stack overflow, freeing all CI structures will
+** force the subsequent call to invoke 'luaE_extendCI', which then
+** will raise any errors.
*/
void luaD_callnoyield (lua_State *L, StkId func, int nResults) {
- L->nny++;
+ incXCcalls(L);
+ if (getCcalls(L) >= LUAI_MAXCCALLS) /* possible stack overflow? */
+ luaE_freeCI(L);
luaD_call(L, func, nResults);
- L->nny--;
+ decXCcalls(L);
}
@@ -530,7 +535,7 @@ static void finishCcall (lua_State *L, int status) {
CallInfo *ci = L->ci;
int n;
/* must have a continuation and must be able to call it */
- lua_assert(ci->u.c.k != NULL && L->nny == 0);
+ lua_assert(ci->u.c.k != NULL && yieldable(L));
/* error status can only happen in a protected call */
lua_assert((ci->callstatus & CIST_YPCALL) || status == LUA_YIELD);
if (ci->callstatus & CIST_YPCALL) { /* was inside a pcall? */
@@ -601,7 +606,6 @@ static int recover (lua_State *L, int status) {
luaD_seterrorobj(L, status, oldtop);
L->ci = ci;
L->allowhook = getoah(ci->callstatus); /* restore original 'allowhook' */
- L->nny = 0; /* should be zero to be yieldable */
luaD_shrinkstack(L);
L->errfunc = ci->u.c.old_errfunc;
return 1; /* continue running the coroutine */
@@ -623,13 +627,6 @@ static int resume_error (lua_State *L, const char *msg, int narg) {
/*
-** "Cost" in the C stack for a coroutine invocation.
-*/
-#if !defined(LUAL_COROCSTK)
-#define LUAL_COROCSTK 3
-#endif
-
-/*
** Do the work for 'lua_resume' in protected mode. Most of the work
** depends on the status of the coroutine: initial state, suspended
** inside a hook, or regularly suspended (optionally with a continuation
@@ -664,7 +661,6 @@ static void resume (lua_State *L, void *ud) {
LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
int *nresults) {
int status;
- unsigned short oldnny = L->nny; /* save "number of non-yieldable" calls */
lua_lock(L);
if (L->status == LUA_OK) { /* may be starting a coroutine */
if (L->ci != &L->base_ci) /* not in base level? */
@@ -675,11 +671,10 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
if (from == NULL)
L->nCcalls = 1;
else /* correct 'nCcalls' for this thread */
- L->nCcalls = from->nCcalls - from->nci + L->nci + LUAL_COROCSTK;
+ L->nCcalls = getCcalls(from) - from->nci + L->nci + CSTACKCF;
if (L->nCcalls >= LUAI_MAXCCALLS)
return resume_error(L, "C stack overflow", nargs);
luai_userstateresume(L, nargs);
- L->nny = 0; /* allow yields */
api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
status = luaD_rawrunprotected(L, resume, &nargs);
/* continue running after recoverable errors */
@@ -698,14 +693,13 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
}
*nresults = (status == LUA_YIELD) ? L->ci->u2.nyield
: cast_int(L->top - (L->ci->func + 1));
- L->nny = oldnny; /* restore 'nny' */
lua_unlock(L);
return status;
}
LUA_API int lua_isyieldable (lua_State *L) {
- return (L->nny == 0);
+ return yieldable(L);
}
@@ -715,7 +709,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
luai_userstateyield(L, nresults);
lua_lock(L);
api_checknelems(L, nresults);
- if (unlikely(L->nny > 0)) {
+ if (unlikely(!yieldable(L))) {
if (L != G(L)->mainthread)
luaG_runerror(L, "attempt to yield across a C-call boundary");
else
@@ -741,7 +735,7 @@ LUA_API int lua_yieldk (lua_State *L, int nresults, lua_KContext ctx,
/*
** Call the C function 'func' in protected mode, restoring basic
-** thread information ('allowhook', 'nny', etc.) and in particular
+** thread information ('allowhook', etc.) and in particular
** its stack level in case of errors.
*/
int luaD_pcall (lua_State *L, Pfunc func, void *u,
@@ -749,7 +743,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
int status;
CallInfo *old_ci = L->ci;
lu_byte old_allowhooks = L->allowhook;
- unsigned short old_nny = L->nny;
ptrdiff_t old_errfunc = L->errfunc;
L->errfunc = ef;
status = luaD_rawrunprotected(L, func, u);
@@ -757,7 +750,6 @@ int luaD_pcall (lua_State *L, Pfunc func, void *u,
StkId oldtop = restorestack(L, old_top);
L->ci = old_ci;
L->allowhook = old_allowhooks;
- L->nny = old_nny;
status = luaF_close(L, oldtop, status);
oldtop = restorestack(L, old_top); /* previous call may change stack */
luaD_seterrorobj(L, status, oldtop);
@@ -811,7 +803,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
const char *mode) {
struct SParser p;
int status;
- L->nny++; /* cannot yield during parsing */
+ incnny(L); /* cannot yield during parsing */
p.z = z; p.name = name; p.mode = mode;
p.dyd.actvar.arr = NULL; p.dyd.actvar.size = 0;
p.dyd.gt.arr = NULL; p.dyd.gt.size = 0;
@@ -822,7 +814,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name,
luaM_freearray(L, p.dyd.actvar.arr, p.dyd.actvar.size);
luaM_freearray(L, p.dyd.gt.arr, p.dyd.gt.size);
luaM_freearray(L, p.dyd.label.arr, p.dyd.label.size);
- L->nny--;
+ decnny(L);
return status;
}
diff --git a/llimits.h b/llimits.h
@@ -184,11 +184,13 @@ typedef LUAI_UACINT l_uacInt;
** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h)
*/
#if LUAI_BITSINT >= 32
-typedef unsigned int Instruction;
+typedef unsigned int l_uint32;
#else
-typedef unsigned long Instruction;
+typedef unsigned long l_uint32;
#endif
+typedef l_uint32 Instruction;
+
/*
diff --git a/lparser.c b/lparser.c
@@ -367,10 +367,12 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) {
}
-#define enterlevel(ls) luaE_incCcalls((ls)->L)
-
+/*
+** Macros to limit the maximum recursion depth while parsing
+*/
+#define enterlevel(ls) luaE_enterCcall((ls)->L)
-#define leavelevel(ls) ((ls)->L->nCcalls--)
+#define leavelevel(ls) luaE_exitCcall((ls)->L)
/*
diff --git a/lstate.c b/lstate.c
@@ -99,24 +99,42 @@ void luaE_setdebt (global_State *g, l_mem debt) {
/*
** Increment count of "C calls" and check for overflows. In case of
** a stack overflow, check appropriate error ("regular" overflow or
-** overflow while handling stack overflow). If 'nCalls' is larger than
-** LUAI_MAXCCALLS (which means it is handling a "regular" overflow) but
-** smaller than 9/8 of LUAI_MAXCCALLS, does not report an error (to
-** allow overflow handling to work)
+** overflow while handling stack overflow).
+** If 'nCcalls' is larger than LUAI_MAXCCALLS but smaller than
+** LUAI_MAXCCALLS + CSTACKCF (plus 2 to avoid by-one errors), it means
+** it has just entered the "overflow zone", so the function raises an
+** overflow error.
+** If 'nCcalls' is larger than LUAI_MAXCCALLS + CSTACKCF + 2
+** (which means it is already handling an overflow) but smaller than
+** 9/8 of LUAI_MAXCCALLS, does not report an error (to allow message
+** handling to work).
+** Otherwise, report a stack overflow while handling a stack overflow
+** (probably caused by a repeating error in the message handling
+** function).
*/
-void luaE_incCcalls (lua_State *L) {
- if (++L->nCcalls >= LUAI_MAXCCALLS) {
- if (L->nCcalls == LUAI_MAXCCALLS)
- luaG_runerror(L, "C stack overflow");
- else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS>>3)))
- luaD_throw(L, LUA_ERRERR); /* error while handing stack error */
+void luaE_enterCcall (lua_State *L) {
+ int ncalls = getCcalls(L);
+ L->nCcalls++;
+ if (ncalls >= LUAI_MAXCCALLS) { /* possible overflow? */
+ luaE_freeCI(L); /* release unused CIs */
+ ncalls = getCcalls(L); /* update call count */
+ if (ncalls >= LUAI_MAXCCALLS) { /* still overflow? */
+ if (ncalls <= LUAI_MAXCCALLS + CSTACKCF + 2) {
+ /* no error before increments; raise the error now */
+ L->nCcalls += (CSTACKCF + 4); /* avoid raising it again */
+ luaG_runerror(L, "C stack overflow");
+ }
+ else if (ncalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS >> 3)))
+ luaD_throw(L, LUA_ERRERR); /* error while handling stack error */
+ }
}
}
CallInfo *luaE_extendCI (lua_State *L) {
CallInfo *ci;
- luaE_incCcalls(L);
+ lua_assert(L->ci->next == NULL);
+ luaE_enterCcall(L);
ci = luaM_new(L, CallInfo);
lua_assert(L->ci->next == NULL);
L->ci->next = ci;
@@ -135,13 +153,13 @@ void luaE_freeCI (lua_State *L) {
CallInfo *ci = L->ci;
CallInfo *next = ci->next;
ci->next = NULL;
- L->nCcalls -= L->nci; /* to subtract removed elements from 'nCcalls' */
+ L->nCcalls -= L->nci; /* subtract removed elements from 'nCcalls' */
while ((ci = next) != NULL) {
next = ci->next;
luaM_free(L, ci);
L->nci--;
}
- L->nCcalls += L->nci; /* to subtract removed elements from 'nCcalls' */
+ L->nCcalls += L->nci; /* adjust result */
}
@@ -151,7 +169,7 @@ void luaE_freeCI (lua_State *L) {
void luaE_shrinkCI (lua_State *L) {
CallInfo *ci = L->ci;
CallInfo *next2; /* next's next */
- L->nCcalls -= L->nci; /* to subtract removed elements from 'nCcalls' */
+ L->nCcalls -= L->nci; /* subtract removed elements from 'nCcalls' */
/* while there are two nexts */
while (ci->next != NULL && (next2 = ci->next->next) != NULL) {
luaM_free(L, ci->next); /* free next */
@@ -160,7 +178,7 @@ void luaE_shrinkCI (lua_State *L) {
next2->previous = ci;
ci = next2; /* keep next's next */
}
- L->nCcalls += L->nci; /* to subtract removed elements from 'nCcalls' */
+ L->nCcalls += L->nci; /* adjust result */
}
@@ -250,7 +268,6 @@ static void preinit_thread (lua_State *L, global_State *g) {
L->allowhook = 1;
resethookcount(L);
L->openupval = NULL;
- L->nny = 1;
L->status = LUA_OK;
L->errfunc = 0;
}
diff --git a/lstate.h b/lstate.h
@@ -15,7 +15,6 @@
/*
-
** Some notes about garbage-collected objects: All objects in Lua must
** be kept somehow accessible until being freed, so all objects always
** belong to one (and only one) of these lists, using field 'next' of
@@ -43,26 +42,58 @@
** 'weak': tables with weak values to be cleared;
** 'ephemeron': ephemeron tables with white->white entries;
** 'allweak': tables with weak keys and/or weak values to be cleared.
-
*/
-/*
+/*
** About 'nCcalls': each thread in Lua (a lua_State) keeps a count of
-** how many "C calls" it has in the C stack, to avoid C-stack overflow.
+** how many "C calls" it can do in the C stack, to avoid C-stack overflow.
** This count is very rough approximation; it considers only recursive
** functions inside the interpreter, as non-recursive calls can be
** considered using a fixed (although unknown) amount of stack space.
**
+** The count itself has two parts: the lower part is the count itself;
+** the higher part counts the number of non-yieldable calls in the stack.
+**
+** Because calls to external C functions can use of unkown amount
+** of space (e.g., functions using an auxiliary buffer), calls
+** to these functions add more than one to the count.
+**
** The proper count also includes the number of CallInfo structures
** allocated by Lua, as a kind of "potential" calls. So, when Lua
** calls a function (and "consumes" one CallInfo), it needs neither to
** increment nor to check 'nCcalls', as its use of C stack is already
** accounted for.
-
*/
+/* number of "C stack slots" used by an external C function */
+#define CSTACKCF 10
+
+/* true if this thread does not have non-yieldable calls in the stack */
+#define yieldable(L) (((L)->nCcalls & 0xffff0000) == 0)
+
+/* real number of C calls */
+#define getCcalls(L) ((L)->nCcalls & 0xffff)
+
+
+/* Increment the number of non-yieldable calls */
+#define incnny(L) ((L)->nCcalls += 0x10000)
+
+/* Decrement the number of non-yieldable calls */
+#define decnny(L) ((L)->nCcalls -= 0x10000)
+
+/* Increment the number of non-yieldable calls and nCcalls */
+#define incXCcalls(L) ((L)->nCcalls += 0x10000 + CSTACKCF)
+
+/* Decrement the number of non-yieldable calls and nCcalls */
+#define decXCcalls(L) ((L)->nCcalls -= 0x10000 + CSTACKCF)
+
+
+
+
+
+
struct lua_longjmp; /* defined in ldo.c */
@@ -208,8 +239,9 @@ typedef struct global_State {
*/
struct lua_State {
CommonHeader;
- unsigned short nci; /* number of items in 'ci' list */
lu_byte status;
+ lu_byte allowhook;
+ unsigned short nci; /* number of items in 'ci' list */
StkId top; /* first free slot in the stack */
global_State *l_G;
CallInfo *ci; /* call info for current function */
@@ -223,13 +255,11 @@ struct lua_State {
CallInfo base_ci; /* CallInfo for first level (C calling Lua) */
volatile lua_Hook hook;
ptrdiff_t errfunc; /* current error handling function (stack index) */
+ l_uint32 nCcalls; /* number of allowed nested C calls - 'nci' */
int stacksize;
int basehookcount;
int hookcount;
- unsigned short nny; /* number of non-yieldable calls in stack */
- unsigned short nCcalls; /* number of nested C calls + 'nny' */
l_signalT hookmask;
- lu_byte allowhook;
};
@@ -283,8 +313,10 @@ LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1);
LUAI_FUNC CallInfo *luaE_extendCI (lua_State *L);
LUAI_FUNC void luaE_freeCI (lua_State *L);
LUAI_FUNC void luaE_shrinkCI (lua_State *L);
-LUAI_FUNC void luaE_incCcalls (lua_State *L);
+LUAI_FUNC void luaE_enterCcall (lua_State *L);
+
+#define luaE_exitCcall(L) ((L)->nCcalls--)
#endif
diff --git a/ltests.h b/ltests.h
@@ -31,7 +31,7 @@
/* compiled with -O0, Lua uses a lot of C stack space... */
#undef LUAI_MAXCCALLS
-#define LUAI_MAXCCALLS 200
+#define LUAI_MAXCCALLS 400
/* to avoid warnings, and to make sure value is really unused */
#define UNUSED(x) (x=0, (void)(x))
diff --git a/luaconf.h b/luaconf.h
@@ -695,14 +695,14 @@
/*
@@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system.
** CHANGE it if it uses too much C-stack space. (For long double,
-** 'string.format("%.99f", -1e4932)' needs 5034 bytes, so a
+** 'string.format("%.99f", -1e4932)' needs 5052 bytes, so a
** smaller buffer would force a memory allocation for each call to
** 'string.format'.)
*/
#if LUA_FLOAT_TYPE == LUA_FLOAT_LONGDOUBLE
#define LUAL_BUFFERSIZE 8192
#else
-#define LUAL_BUFFERSIZE ((int)(0x80 * sizeof(void*) * sizeof(lua_Integer)))
+#define LUAL_BUFFERSIZE ((int)(16 * sizeof(void*) * sizeof(lua_Number)))
#endif
/*
diff --git a/testes/all.lua b/testes/all.lua
@@ -172,6 +172,7 @@ if not _G._soft then
assert(f() == 'b')
assert(f() == 'a')
end
+dofile('cstack.lua')
dofile('nextvar.lua')
dofile('pm.lua')
dofile('utf8.lua')
diff --git a/testes/coroutine.lua b/testes/coroutine.lua
@@ -107,7 +107,7 @@ function filter (p, g)
end)
end
-local x = gen(100)
+local x = gen(80)
local a = {}
while 1 do
local n = x()
@@ -116,7 +116,7 @@ while 1 do
x = filter(n, x)
end
-assert(#a == 25 and a[#a] == 97)
+assert(#a == 22 and a[#a] == 79)
x, a = nil
diff --git a/testes/cstack.lua b/testes/cstack.lua
@@ -0,0 +1,62 @@
+-- $Id: testes/cstack.lua $
+-- See Copyright Notice in file all.lua
+
+print"testing C-stack overflow detection"
+
+-- Segmentation faults in these tests probably result from a C-stack
+-- overflow. To avoid these errors, recompile Lua with a smaller
+-- value for the constant 'LUAI_MAXCCALLS' or else ensure a larger
+-- stack for the program.
+
+local function checkerror (msg, f, ...)
+ local s, err = pcall(f, ...)
+ assert(not s and string.find(err, msg))
+end
+
+
+do -- simple recursion
+ local count = 0
+ local function foo ()
+ count = count + 1
+ foo()
+ end
+ checkerror("stack overflow", foo)
+ print(" maximum recursion: " .. count)
+end
+
+
+-- bug since 2.5 (C-stack overflow in recursion inside pattern matching)
+do
+ local function f (size)
+ local s = string.rep("a", size)
+ local p = string.rep(".?", size)
+ return string.match(s, p)
+ end
+ local m = f(80)
+ assert(#m == 80)
+ checkerror("too complex", f, 200000)
+end
+
+
+-- testing stack-overflow in recursive 'gsub'
+do
+ local count = 0
+ local function foo ()
+ count = count + 1
+ string.gsub("a", ".", foo)
+ end
+ checkerror("stack overflow", foo)
+ print(" maximum 'gsub' nest (calls): " .. count)
+
+ -- can be done with metamethods, too
+ count = 0
+ local t = setmetatable({}, {__index = foo})
+ foo = function ()
+ count = count + 1
+ string.gsub("a", ".", t)
+ end
+ checkerror("stack overflow", foo)
+ print(" maximum 'gsub' nest (metamethods): " .. count)
+end
+
+print'OK'
diff --git a/testes/pm.lua b/testes/pm.lua
@@ -237,18 +237,6 @@ checkerror("invalid capture index %%0", string.gsub, "alo", "(%0)", "a")
checkerror("invalid capture index %%1", string.gsub, "alo", "(%1)", "a")
checkerror("invalid use of '%%'", string.gsub, "alo", ".", "%x")
--- bug since 2.5 (C-stack overflow)
-do
- local function f (size)
- local s = string.rep("a", size)
- local p = string.rep(".?", size)
- return pcall(string.match, s, p)
- end
- local r, m = f(80)
- assert(r and #m == 80)
- r, m = f(200000)
- assert(not r and string.find(m, "too complex"))
-end
if not _soft then
print("big strings")
