Thread scheduling fixes (#722)

* Add thread preemption when a thread wakes up from a timeout

* Add arbiter sleeping with timeout

Update threads.cpp

* Scheduler: Fix event numbers

* More address arbiter fixes

* Update preempt threads branch (#755)

* CI: Fix Vulkan SDK action (#723)

* GPU registers: Fix writes to some registers ignoring the mask (#725)

Co-authored-by: henry <23128103+atem2069@users.noreply.github.com>

* OLED theme

* OLED theme config fix (#736)

Co-authored-by: smiRaphi <neogt404@gmail.com>

* Adding Swedish translation

* Fix Metal renderer compilation on iOS

* [Core] Improve iOS compilation workflow

* [Qt] Hook Swedish to UI

* AppDataDocumentProvider: Typo (#740)

* More iOS work

* More iOS progress

* More iOS work

* AppDataDocumentProvider: Add missing ``COLUMN_FLAGS`` in the default document projectation (#741)

Fixes unable to copy files from device to app's internal storage problem

* More iOS work

* ios: Simplify MTKView interface (still doesn't work though)

* ios: Pass CAMetalLayer instead of void* to Obj-C++ bridging header

* Fix bridging cast

* FINALLY IOS GRAPHICS

* ios: Remove printf spam

* Metal: Reimplement some texture formats on iOS

* metal: implement texture decoder

* metal: check for format support

* metal: implement texture swizzling

* metal: remove unused texture functions

* Shadergen types: Add Metal & MSL

* Format

* Undo submodule changes

* Readme: Add Chonkystation 3

* Metal: Use std::unique_ptr for texture decode

* AppDataDocumentProvider: Allow to remove documents (#744)

* AppDataDocumentProvider: Allow to remove documents

* Typo

* Metal renderer fixes for iOS

* iOS driver: Add doc comments

* iOS: Add frontend & frontend build files (#746)

* iOS: Add SwiftUI part to repo

* Add iOS build script

* Update SDL2 submodule

* Fix iOS build script

* CI: Update xcode tools for iOS

* Update iOS_Build.yml

* Update iOS build

* Lower XCode version

* A

* Update project.pbxproj

* Update iOS_Build.yml

* Update iOS_Build.yml

* Update build.sh

* iOS: Fail on build error

* iOS: Add file picker (#747)

* iOS: Add file picker

* Fix lock placement

* Qt: Add runpog icon (#752)

* Update discord-rpc submodule (#753)

* Remove cryptoppwin submodule (#754)

---------

Co-authored-by: henry <23128103+atem2069@users.noreply.github.com>
Co-authored-by: smiRaphi <neogt404@gmail.com>
Co-authored-by: smiRaphi <87574679+smiRaphi@users.noreply.github.com>
Co-authored-by: Daniel Nylander <po@danielnylander.se>
Co-authored-by: Ishan09811 <156402647+Ishan09811@users.noreply.github.com>
Co-authored-by: Samuliak <samuliak77@gmail.com>
Co-authored-by: Albert <45282415+ggrtk@users.noreply.github.com>

---------

Co-authored-by: henry <23128103+atem2069@users.noreply.github.com>
Co-authored-by: smiRaphi <neogt404@gmail.com>
Co-authored-by: smiRaphi <87574679+smiRaphi@users.noreply.github.com>
Co-authored-by: Daniel Nylander <po@danielnylander.se>
Co-authored-by: Ishan09811 <156402647+Ishan09811@users.noreply.github.com>
Co-authored-by: Samuliak <samuliak77@gmail.com>
Co-authored-by: Albert <45282415+ggrtk@users.noreply.github.com>

include/kernel/kernel.hpp

@@ -62,6 +62,7 @@ class Kernel {
 	// Top 8 bits are the major version, bottom 8 are the minor version
 	u16 kernelVersion = 0;
 
+	u64 nextScheduledWakeupTick = std::numeric_limits<u64>::max();
 	// Shows whether a reschedule will be need
 	bool needReschedule = false;
 
@@ -98,6 +99,8 @@ class Kernel {
 	void signalArbiter(u32 waitingAddress, s32 threadCount);
 	void sleepThread(s64 ns);
 	void sleepThreadOnArbiter(u32 waitingAddress);
+	void sleepThreadOnArbiterWithTimeout(u32 waitingAddress, s64 timeoutNs);
+
 	void switchThread(int newThreadIndex);
 	void sortThreads();
 	std::optional<int> getNextThread();
@@ -218,6 +221,8 @@ class Kernel {
 		}
 	}
 
+	void addWakeupEvent(u64 tick);
+
 	Handle makeObject(KernelObjectType type) {
 		if (handleCounter > KernelHandles::Max) [[unlikely]] {
 			Helpers::panic("Hlep we somehow created enough kernel objects to overflow this thing");
@@ -255,6 +260,8 @@ class Kernel {
 	void sendGPUInterrupt(GPUInterrupt type) { serviceManager.sendGPUInterrupt(type); }
 	void clearInstructionCache();
 	void clearInstructionCacheRange(u32 start, u32 size);
+	void pollThreadWakeups();
+
 	u32 getSharedFontVaddr();
 
 	// For debuggers: Returns information about the main process' (alive) threads in a vector for the frontend to display

include/kernel/kernel_types.hpp

@@ -112,16 +112,17 @@ struct Session {
 };
 
 enum class ThreadStatus {
-	Running,      // Currently running
+	Running,             // Currently running
-	Ready,        // Ready to run
+	Ready,               // Ready to run
-	WaitArbiter,  // Waiting on an address arbiter
+	WaitArbiter,         // Waiting on an address arbiter
-	WaitSleep,    // Waiting due to a SleepThread SVC
+	WaitArbiterTimeout,  // Waiting on an address arbiter with timeout
-	WaitSync1,    // Waiting for the single object in the wait list to be ready
+	WaitSleep,           // Waiting due to a SleepThread SVC
-	WaitSyncAny,  // Wait for one object of the many that might be in the wait list to be ready
+	WaitSync1,           // Waiting for the single object in the wait list to be ready
-	WaitSyncAll,  // Waiting for ALL sync objects in its wait list to be ready
+	WaitSyncAny,         // Wait for one object of the many that might be in the wait list to be ready
-	WaitIPC,      // Waiting for the reply from an IPC request
+	WaitSyncAll,         // Waiting for ALL sync objects in its wait list to be ready
-	Dormant,      // Created but not yet made ready
+	WaitIPC,             // Waiting for the reply from an IPC request
-	Dead          // Run to completion, or forcefully terminated
+	Dormant,             // Created but not yet made ready
+	Dead                 // Run to completion, or forcefully terminated
 };
 
 struct Thread {

include/scheduler.hpp

@@ -8,11 +8,12 @@
 struct Scheduler {
 	enum class EventType {
 		VBlank = 0,          // End of frame event
-		UpdateTimers = 1,    // Update kernel timer objects
+		ThreadWakeup = 1,    // A thread might wake up from eg sleeping or timing outs
 		RunDSP = 2,          // Make the emulated DSP run for one audio frame
-		SignalY2R = 3,       // Signal that a Y2R conversion has finished
+		UpdateTimers = 3,    // Update kernel timer objects
-		UpdateIR = 4,        // Update an IR device (For now, just the CirclePad Pro/N3DS controls)
+		SignalY2R = 4,       // Signal that a Y2R conversion has finished
-		Panic = 5,           // Dummy event that is always pending and should never be triggered (Timestamp = UINT64_MAX)
+		UpdateIR = 5,        // Update an IR device (For now, just the CirclePad Pro/N3DS controls)
+		Panic = 6,           // Dummy event that is always pending and should never be triggered (Timestamp = UINT64_MAX)
 		TotalNumberOfEvents  // How many event types do we have in total?
 	};
 	static constexpr usize totalNumberOfEvents = static_cast<usize>(EventType::TotalNumberOfEvents);
@@ -88,4 +89,4 @@ struct Scheduler {
 
 		return (arm11Clock * s64(ns)) / 1000000000;
 	}
-};
+};

src/core/kernel/address_arbiter.cpp

@@ -80,6 +80,14 @@ void Kernel::arbitrateAddress() {
 			break;
 		}
 
+		case ArbitrationType::WaitIfLessTimeout: {
+			s32 word = static_cast<s32>(mem.read32(address));  // Yes this is meant to be signed
+			if (word < value) {
+				sleepThreadOnArbiterWithTimeout(address, ns);
+			}
+			break;
+		}
+
 		case ArbitrationType::Signal: signalArbiter(address, value); break;
 		default: Helpers::panic("ArbitrateAddress: Unimplemented type %s", arbitrationTypeToString(type));
 	}
@@ -96,8 +104,9 @@ void Kernel::signalArbiter(u32 waitingAddress, s32 threadCount) {
 	// Wake threads with the highest priority threads being woken up first
 	for (auto index : threadIndices) {
 		Thread& t = threads[index];
-		if (t.status == ThreadStatus::WaitArbiter && t.waitingAddress == waitingAddress) {
+		if ((t.status == ThreadStatus::WaitArbiter || t.status == ThreadStatus::WaitArbiterTimeout) && t.waitingAddress == waitingAddress) {
 			t.status = ThreadStatus::Ready;
+			t.gprs[0] = Result::Success;  // Return that the arbiter was actually signalled and that we didn't timeout
 			count += 1;
 
 			// Check if we've reached the max number of. If count < 0 then all threads are released.

src/core/kernel/events.cpp

@@ -138,6 +138,7 @@ void Kernel::waitSynchronization1() {
 		// Add the current thread to the object's wait list
 		object->getWaitlist() |= (1ull << currentThreadIndex);
 
+		addWakeupEvent(t.wakeupTick);
 		requireReschedule();
 	}
 }
@@ -232,6 +233,7 @@ void Kernel::waitSynchronizationN() {
 			waitObjects[i].second->getWaitlist() |= (1ull << currentThreadIndex);  // And add the thread to the object's waitlist
 		}
 
+		addWakeupEvent(t.wakeupTick);
 		requireReschedule();
 	} else {
 		Helpers::panic("WaitSynchronizationN with waitAll");

src/core/kernel/kernel.cpp

@@ -1,6 +1,7 @@
 #include "kernel.hpp"
 
 #include <cassert>
+#include <limits>
 
 #include "cpu.hpp"
 #include "kernel_types.hpp"
@@ -161,6 +162,7 @@ void Kernel::reset() {
 	threadIndices.clear();
 	serviceManager.reset();
 
+	nextScheduledWakeupTick = std::numeric_limits<u64>::max();
 	needReschedule = false;
 
 	// Allocate handle #0 to a dummy object and make a main process object

src/core/kernel/threads.cpp

@@ -2,6 +2,7 @@
 #include <bit>
 #include <cassert>
 #include <cstring>
+#include <limits>
 
 #include "arm_defs.hpp"
 #include "kernel.hpp"
@@ -50,7 +51,7 @@ bool Kernel::canThreadRun(const Thread& t) {
 	if (t.status == ThreadStatus::Ready) {
 		return true;
 	} else if (t.status == ThreadStatus::WaitSleep || t.status == ThreadStatus::WaitSync1 || t.status == ThreadStatus::WaitSyncAny ||
-			   t.status == ThreadStatus::WaitSyncAll) {
+			   t.status == ThreadStatus::WaitSyncAll || t.status == ThreadStatus::WaitArbiterTimeout) {
 		// TODO: Set r0 to the correct error code on timeout for WaitSync{1/Any/All}
 		return cpu.getTicks() >= t.wakeupTick;
 	}
@@ -215,6 +216,23 @@ void Kernel::sleepThreadOnArbiter(u32 waitingAddress) {
 	requireReschedule();
 }
 
+void Kernel::sleepThreadOnArbiterWithTimeout(u32 waitingAddress, s64 timeoutNs) {
+	regs[0] = Result::OS::Timeout;  // This will be overwritten with success if we don't timeout
+
+	// Timeout is 0, don't bother waiting, instantly timeout
+	if (timeoutNs == 0) {
+		return;
+	}
+
+	Thread& t = threads[currentThreadIndex];
+	t.status = ThreadStatus::WaitArbiterTimeout;
+	t.waitingAddress = waitingAddress;
+	t.wakeupTick = getWakeupTick(timeoutNs);
+
+	addWakeupEvent(t.wakeupTick);
+	requireReschedule();
+}
+
 // Acquires an object that is **ready to be acquired** without waiting on it
 void Kernel::acquireSyncObject(KernelObject* object, const Thread& thread) {
 	switch (object->type) {
@@ -390,6 +408,7 @@ void Kernel::sleepThread(s64 ns) {
 		t.status = ThreadStatus::WaitSleep;
 		t.wakeupTick = getWakeupTick(ns);
 
+		addWakeupEvent(t.wakeupTick);
 		requireReschedule();
 	}
 }
@@ -688,6 +707,42 @@ bool Kernel::shouldWaitOnObject(KernelObject* object) {
 	}
 }
 
+void Kernel::pollThreadWakeups() {
+	rescheduleThreads();
+	bool haveSleepingThread = false;
+	u64 nextWakeupTick = std::numeric_limits<u64>::max();
+
+	for (auto index : threadIndices) {
+		const Thread& t = threads[index];
+
+		if (t.status == ThreadStatus::WaitSleep || t.status == ThreadStatus::WaitSync1 || t.status == ThreadStatus::WaitSyncAny ||
+			t.status == ThreadStatus::WaitSyncAll || t.status == ThreadStatus::WaitArbiterTimeout) {
+			nextWakeupTick = std::min<u64>(nextWakeupTick, t.wakeupTick);
+			haveSleepingThread = true;
+		}
+	}
+
+	auto& scheduler = cpu.getScheduler();
+
+	if (haveSleepingThread && nextWakeupTick > scheduler.currentTimestamp) {
+		nextScheduledWakeupTick = nextWakeupTick;
+		scheduler.addEvent(Scheduler::EventType::ThreadWakeup, nextWakeupTick);
+	} else {
+		nextScheduledWakeupTick = std::numeric_limits<u64>::max();
+	}
+}
+
+void Kernel::addWakeupEvent(u64 tick) {
+	// We only need to queue the event if the tick of the wakeup is coming sooner than our next scheduled wakeup.
+	if (nextScheduledWakeupTick > tick) {
+		nextScheduledWakeupTick = tick;
+		auto& scheduler = cpu.getScheduler();
+
+		scheduler.removeEvent(Scheduler::EventType::ThreadWakeup);
+		scheduler.addEvent(Scheduler::EventType::ThreadWakeup, tick);
+	}
+}
+
 std::vector<Thread> Kernel::getMainProcessThreads() {
 	// Sort the thread indices so that they appear nicer in the debugger
 	auto indices = threadIndices;

src/emulator.cpp

@@ -175,6 +175,7 @@ void Emulator::pollScheduler() {
 					break;
 				}
 
+			case Scheduler::EventType::ThreadWakeup: kernel.pollThreadWakeups(); break;
 			case Scheduler::EventType::UpdateTimers: kernel.pollTimers(); break;
 			case Scheduler::EventType::RunDSP: {
 				dsp->runAudioFrame(time);