luzer: fix memory leak in FDP

CHANGELOG.md

@@ -61,3 +61,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - An error when Clang couldn't find the library.
 - A possible corruption on copying library (#91).
 - A crash due to incorrect use of signal handler (#40).
+- Memory leak in FuzzedDataProvider (#52).
+- Segfault on parsing a broken dictionary (#65).

docs/usage.md

@@ -228,9 +228,33 @@ and can be disabled by setting the enviroment variable
 `DISABLE_LUAJIT_METRICS`. Learn more about the enviroment variable
 in the section [LuaJIT Metrics](#luajit-metrics).
 
+### Memory leaks
+
+When a target application (or a fuzzer) consumes increasing
+amounts of RAM over time without releasing it, it can be a normal
+memory consumption or memory leak is occurring. The common causes
+of memory leak are: unreleased references, improper handling of
+native resources in Lua. If you are encountering a memory leak in
+a target application while using luzer, you may need to use memory
+debugging tools to identify the specific code segment that isn't
+freeing memory.
+
+luzer can encounter false positive memory leaks during testing.
+When fuzzing native extensions, using FDP (FuzzingDataProvider), or
+using FFI memory leak detection (ASan) should be disabled to
+prevent false reports. Set flag `-detect_leaks=0` using enviroment
+variable [`ASAN_OPTIONS`][asan-flags] as it is recommended by
+AddressSanitizer:
+
+```
+SUMMARY: AddressSanitizer: 96 byte(s) leaked in 6 allocation(s).
+INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.
+```
+
 [ffi-library-url]: https://luajit.org/ext_ffi.html
 [programming-in-lua-8]: https://www.lua.org/pil/8.html
 [programming-in-lua-24]: https://www.lua.org/pil/24.html
 [atheris-native-extensions]: https://github.com/google/atheris/blob/master/native_extension_fuzzing.md
 [atheris-native-extensions-video]: https://www.youtube.com/watch?v=oM-7lt43-GA
 [luacov-website]: https://lunarmodules.github.io/luacov/
+[asan-flags]: https://github.com/google/sanitizers/wiki/addresssanitizerflags

luzer/fuzzed_data_provider.cc

@@ -27,6 +27,52 @@ int table_nkeys(lua_State *L, int idx);
  */
 #define FDP_LUA_UDATA_NAME	"fdp"
 
+#define FDP_REFS_TABLE	"LUZER_FDP_REFS"
+
+#if LUA_VERSION_NUM == 501
+#define lua_rawlen(L, idx)	lua_objlen((L), (idx))
+#endif
+
+NO_SANITIZE int
+ref_fdp(lua_State *L) {
+	luaL_checktype(L, -1, LUA_TUSERDATA);
+	lua_pushvalue(L, -1);
+	int ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
+	lua_getfield(L, LUA_REGISTRYINDEX, FDP_REFS_TABLE);
+	if (lua_isnil(L, -1)) {
+		lua_pop(L, 1);
+		lua_newtable(L);
+		lua_pushvalue(L, -1);
+		lua_setfield(L, LUA_REGISTRYINDEX, FDP_REFS_TABLE);
+	}
+	int n = lua_rawlen(L, -1);
+	lua_pushinteger(L, ref);
+	lua_rawseti(L, -2, n + 1);
+	lua_pop(L, 1);
+
+	return 0;
+}
+
+NO_SANITIZE void
+unref_fdp(lua_State *L) {
+	lua_getfield(L, LUA_REGISTRYINDEX, FDP_REFS_TABLE);
+	if (lua_isnil(L, -1)) {
+		lua_pop(L, 1);
+		return;
+	}
+	int n = lua_rawlen(L, -1);
+	for (int i = 1; i <= n; i++) {
+		lua_rawgeti(L, -1, i);
+		int ref = lua_tointeger(L, -1);
+		luaL_unref(L, LUA_REGISTRYINDEX, ref);
+		lua_pop(L, 1);
+	}
+	lua_pushnil(L);
+	lua_setfield(L, LUA_REGISTRYINDEX, FDP_REFS_TABLE);
+	lua_pop(L, 1);
+}
+
 /*
  * A convenience wrapper turning the raw fuzzer input bytes into Lua primitive
  * types. The methods behave similarly to math.random(), with all returned
@@ -307,5 +353,7 @@ luaL_fuzzed_data_provider(lua_State *L)
 	luaL_getmetatable(L, FDP_LUA_UDATA_NAME);
 	lua_setmetatable(L, -2);
 
+	ref_fdp(L);
+
 	return 1;
 }

luzer/fuzzed_data_provider.h

@@ -4,6 +4,9 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+	int ref_fdp(lua_State *L);
+	void unref_fdp(lua_State *L);
+
 	void fdp_metatable_init(lua_State *L);
 	int luaL_fuzzed_data_provider(lua_State *L);
 #ifdef __cplusplus

luzer/init.lua

@@ -68,7 +68,20 @@ local function Fuzz(test_one_input, custom_mutator, func_args)
     if type(luzer_args) ~= "table" then
         error("args is not a table")
     end
+
     local flags = build_flags(arg, luzer_args)
+
+    -- Lua has the GC-based memory management model, it may
+    -- accumulate memory between the TestOneInput() runs.
+    -- libFuzzer has a flag `detect_leaks`, it is enabled by
+    -- default, when the option is enabled and if LeakSanitizer is
+    -- enabled it tries to detect memory leaks during fuzzing
+    -- (i.e. not only at shut down). The option `detect_leaks` may
+    -- lead to false positives, so it is disabled by default.
+    if flags["detect_leaks"] == nil then
+        flags["detect_leaks"] = 0
+    end
+
     local test_path = arg[0]
     local lua_bin = progname(arg)
     local test_cmd = ("%s %s"):format(lua_bin, test_path)

luzer/luzer.c

@@ -151,9 +151,9 @@ get_symbol_path(void *addr) {
 	return path;
 }
 
-const char *dso_path_lf_asan;
-const char *dso_path_lf_ubsan;
-const char *dso_path_libcustom_mutator;
+char *dso_path_lf_asan;
+char *dso_path_lf_ubsan;
+char *dso_path_libcustom_mutator;
 
 NO_SANITIZE void
 init(void)
@@ -290,6 +290,59 @@ teardown(void)
 	metrics_print();
 }
 
+NO_SANITIZE static int
+luzer_cleanup(lua_State *L)
+{
+	lua_pushnil(L);
+	lua_setglobal(L, TEST_ONE_INPUT_FUNC);
+	lua_pushnil(L);
+	lua_setglobal(L, DEBUG_HOOK_FUNC);
+	lua_pushnil(L);
+	lua_setglobal(L, CUSTOM_MUTATOR_FUNC);
+	unref_fdp(L);
+
+	free(dso_path_lf_asan);
+	dso_path_lf_asan = NULL;
+	free(dso_path_lf_ubsan);
+	dso_path_lf_ubsan = NULL;
+	free(dso_path_libcustom_mutator);
+	dso_path_libcustom_mutator = NULL;
+
+	return 0;
+}
+
+NO_SANITIZE static void
+shutdown_lua(void)
+{
+	lua_State *L = get_global_lua_state();
+	luzer_cleanup(L);
+	lua_settop(L, 0);
+	lua_gc(L, LUA_GCCOLLECT, 0);
+	lua_gc(L, LUA_GCCOLLECT, 0);
+	lua_close(L);
+	set_global_lua_state(NULL);
+}
+
+int atexit_retcode;
+
+NO_SANITIZE void
+atexit_handler(void)
+{
+	_exit(atexit_retcode);
+}
+
+NO_SANITIZE static void
+graceful_exit(int retcode, bool prevent_crash_report)
+{
+	if (prevent_crash_report) {
+		/* Override libFuzzer's atexit(). */
+		atexit_retcode = retcode;
+		atexit(&atexit_handler);
+	}
+	shutdown_lua();
+	exit(retcode);
+}
+
 NO_SANITIZE int
 TestOneInput(const uint8_t* data, size_t size) {
 	const counter_and_pc_table_range alloc = allocate_counters_and_pcs();
@@ -346,18 +399,6 @@ TestOneInput(const uint8_t* data, size_t size) {
 	return rc;
 }
 
-NO_SANITIZE static int
-luaL_cleanup(lua_State *L)
-{
-	lua_pushnil(L);
-	lua_setglobal(L, TEST_ONE_INPUT_FUNC);
-	lua_pushnil(L);
-	lua_setglobal(L, DEBUG_HOOK_FUNC);
-	lua_pushnil(L);
-	lua_setglobal(L, CUSTOM_MUTATOR_FUNC);
-	return 0;
-}
-
 NO_SANITIZE static int
 luaL_path(lua_State *L) {
 	lua_createtable(L, 0, 3);
@@ -535,13 +576,10 @@ luaL_fuzz(lua_State *L)
 #endif
 	set_global_lua_state(L);
 	int rc = LLVMFuzzerRunDriver(&argc, &argv, &TestOneInput);
-
 	free_argv(argc, argv);
-	luaL_cleanup(L);
-
-	lua_pushnumber(L, rc);
-
-	return 1;
+	free((void *)corpus_path);
+	graceful_exit(rc, true);
+	return 0;
 }
 
 static const struct luaL_Reg Module[] = {

luzer/tests/CMakeLists.txt

@@ -363,6 +363,11 @@ macro(generate_ffi_test name env_vars pass_regex)
   )
 endmacro()
 
+list(APPEND LSAN_REGULAR_EXPRESSION
+  "LeakSanitizer: detected memory leaks"
+  "[0-9]+ byte(s) leaked in [0-9]+ allocation"
+)
+
 list(APPEND TEST_ENV
   "LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR};"
 )
@@ -376,10 +381,7 @@ if (LUA_HAS_JIT)
     LD_PRELOAD=${ASAN_DSO_PATH}
     FFI_LIB_NAME=testlib_asan${CMAKE_SHARED_LIBRARY_SUFFIX}
   )
-  # XXX: Memory leak in FDP is expected on Linux, should be fixed
-  # in [1].
-  # 1. https://github.com/ligurio/luzer/issues/52
-  set(PASS_PATTERN "LeakSanitizer: detected memory leaks")
+  set(PASS_PATTERN ${LSAN_REGULAR_EXPRESSION})
   if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     set(PASS_PATTERN "Done 10 runs in 0 second")
   endif()
@@ -419,3 +421,45 @@ if (LUA_HAS_JIT)
     "runtime error: load of null pointer of type"
   )
 endif()
+
+# FuzzedDataProvider leak test (issue #52)
+add_test(
+  NAME luzer_leak_fdp
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_leak_fdp.lua
+          -runs=100
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+)
+set_tests_properties(luzer_leak_fdp PROPERTIES
+  ENVIRONMENT "LUA_CPATH=${LUA_CPATH};LUA_PATH=${LUA_PATH};LD_PRELOAD=${ASAN_DSO_PATH}"
+  PASS_REGULAR_EXPRESSION "Done 100 runs in [0-9]+ second"
+)
+
+string(JOIN ";" TEST_ENV
+  "LUA_CPATH=${LUA_CPATH}"
+  "LUA_PATH=${LUA_PATH}"
+  "LD_PRELOAD=${ASAN_DSO_PATH}"
+)
+add_test(
+  NAME luzer_missed_dict_test
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_options_3.lua
+          -dict=${CMAKE_CURRENT_SOURCE_DIR}/nonexistent.dict
+)
+set_tests_properties(luzer_missed_dict_test PROPERTIES
+  ENVIRONMENT "${TEST_ENV}"
+  PASS_REGULAR_EXPRESSION "ParseDictionaryFile: file does not exist or is empty"
+  FAIL_REGULAR_EXPRESSION "${LSAN_REGULAR_EXPRESSION}"
+)
+
+add_test(
+  NAME luzer_leak_memory_test
+  COMMAND ${LUA_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/leak_memory.lua
+          -rss_limit_mb=500
+)
+list(APPEND PASS_REGULAR_EXPRESSION
+  "libFuzzer disabled leak detection after every mutation"
+  "ERROR: libFuzzer: out-of-memory"
+)
+set_tests_properties(luzer_leak_memory_test PROPERTIES
+  ENVIRONMENT "${TEST_ENV}"
+  PASS_REGULAR_EXPRESSION "${PASS_REGULAR_EXPRESSION}"
+)

luzer/tests/leak_memory.lua

@@ -0,0 +1,21 @@
+local luzer = require("luzer")
+
+local leaky_cache = {} -- luacheck: no unused
+
+-- Function to create a large, nested table.
+local function make_big_object(size)
+    if size <= 0 then
+        return {}
+    end
+    -- Create nested tables recursively.
+    return {
+        make_big_object(size - 1)
+    }
+end
+
+local function TestOneInput(_buf)
+    local new_object = make_big_object(2)
+    table.insert(leaky_cache, new_object)
+end
+
+luzer.Fuzz(TestOneInput)

luzer/tests/test_leak_fdp.lua

@@ -0,0 +1,9 @@
+local luzer = require("luzer")
+
+local function TestOneInput(buf)
+    local fdp = luzer.FuzzedDataProvider(buf)
+    fdp:consume_string(1)
+end
+
+local opts = { runs = 100 }
+luzer.Fuzz(TestOneInput, nil, opts)

luzer/tests/test_options_3.lua

@@ -0,0 +1,10 @@
+local luzer = require("luzer")
+
+local function TestOneInput(buf)
+    local fdp = luzer.FuzzedDataProvider(buf)
+    local str = fdp:consume_string(100)
+    local str_chars = {}
+    str:gsub(".", function(c) table.insert(str_chars, c) end)
+end
+
+luzer.Fuzz(TestOneInput)