Upgrade Lua service intercepts (#775)

Co-authored-by: Théo B. <16072534+LiquidFenrir@users.noreply.github.com>

src/lua.cpp

@@ -101,14 +101,14 @@ void LuaManager::signalEventInternal(LuaEvent e) {
 	lua_pcall(L, 1, 0, 0);
 }
 
-// Calls the "interceptService" function, if it exists, when a service call is intercepted
+// Calls the callback passed to the addServiceIntercept function when a service call is intercepted
 // It passes the service name, the function header, and a pointer to the call's TLS buffer as parameters
-// interceptService is expected to return a bool, which indicates whether the C++ code should proceed to handle the service call
+// The callback is expected to return a bool, indicating whether the C++ code should proceed to handle the service call
 // or if the Lua code handles it entirely.
-// If the bool is true, the Lua code handles the service call entirely and the C++ code doesn't do anything extra
-// Otherwise, then the C++ code calls its service call handling code as usual.
-bool LuaManager::signalInterceptedService(const std::string& service, u32 function, u32 messagePointer) {
-	lua_getglobal(L, "interceptService");
+// If the bool is true, the Lua code handles the service call entirely and the C++ side doesn't do anything extra
+// Otherwise, the C++ side calls its service call handling code as usual.
+bool LuaManager::signalInterceptedService(const std::string& service, u32 function, u32 messagePointer, int callbackRef) {
+	lua_rawgeti(L, LUA_REGISTRYINDEX, callbackRef);
 	lua_pushstring(L, service.c_str());  // Push service name
 	lua_pushinteger(L, function);        // Push function header
 	lua_pushinteger(L, messagePointer);  // Push pointer to TLS buffer
@@ -129,6 +129,12 @@ bool LuaManager::signalInterceptedService(const std::string& service, u32 functi
 	return ret;
 }
 
+// Removes a reference from the callback value in the registry
+// Prevents memory leaks, otherwise the function object would stay forever
+void LuaManager::removeInterceptedService(const std::string& service, u32 function, int callbackRef) {
+	luaL_unref(L, LUA_REGISTRYINDEX, callbackRef);
+}
+
 void LuaManager::reset() {
 	// Reset scripts
 	haveScript = false;
@@ -238,15 +244,20 @@ static int loadROMThunk(lua_State* L) {
 static int addServiceInterceptThunk(lua_State* L) {
 	// Service name argument is invalid, report that loading failed and exit
 	if (lua_type(L, 1) != LUA_TSTRING) {
-		lua_pushboolean(L, 0);
-		lua_error(L);
-		return 2;
+		return luaL_error(L, "Argument 1 (service name) is not a string");
 	}
 
 	if (lua_type(L, 2) != LUA_TNUMBER) {
-		lua_pushboolean(L, 0);
-		lua_error(L);
-		return 2;
+		return luaL_error(L, "Argument 2 (function id) is not a number");
+	}
+
+	// Callback is not a function object directly, fail and exit
+	// Objects with a __call metamethod are not allowed (tables, userdata)
+	// Good: addServiceIntercept(serviceName, func, myLuaFunction)
+	// Good: addServiceIntercept(serviceName, func, function (service, func, buffer) ... end)
+	// Bad:  addServiceIntercept(serviceName, func, obj:method)
+	if (lua_type(L, 3) != LUA_TFUNCTION) {
+		return luaL_error(L, "Argument 3 (callback) is not a function");
 	}
 
 	// Get the name of the service we want to intercept, as well as the header of the function to intercept
@@ -254,8 +265,13 @@ static int addServiceInterceptThunk(lua_State* L) {
 	const char* const str = lua_tolstring(L, 1, &nameLength);
 	const u32 function = (u32)lua_tointeger(L, 2);
 	const auto serviceName = std::string(str, nameLength);
-	LuaManager::g_emulator->getServiceManager().addServiceIntercept(serviceName, function);
-	return 2;
+
+	// Stores a reference to the callback function object in the registry for later use
+	// Must be freed with lua_unref later, in order to avoid memory leaks
+	lua_pushvalue(L, 3);
+	const int callbackRef = luaL_ref(L, LUA_REGISTRYINDEX);
+	LuaManager::g_emulator->getServiceManager().addServiceIntercept(serviceName, function, callbackRef);
+	return 0;
 }
 
 static int clearServiceInterceptsThunk(lua_State* L) {
@@ -391,7 +407,7 @@ void LuaManager::initializeThunks() {
 
 		disassembleARM = function(pc, instruction) return GLOBALS.__disassembleARM(pc, instruction) end,
 		disassembleTeak = function(opcode, exp) return GLOBALS.__disassembleTeak(opcode, exp or 0) end,
-		addServiceIntercept = function(service, func) return GLOBALS.__addServiceIntercept(service, func) end,
+		addServiceIntercept = function(service, func, cb) return GLOBALS.__addServiceIntercept(service, func, cb) end,
 		clearServiceIntercepts = function() return GLOBALS.__clearServiceIntercepts() end,
 
 		Frame = __Frame,