Add Metal support to Qt frontend and clean up renderer creation code (#795)

* Qt: Initial support for Metal renderer

* Clean up graphics context code

* Nits

* More nits

* Qt: Move screen-related stuff to own folder

* Qt: Make screen widget polymorphic

* Qt: Re-add Metal

* Add factory for screen widget

* Qt: Support compilation without Metal

* Qt: Fix build without Metal

* Oops

* oops

include/PICA/gpu.hpp

@@ -109,11 +109,7 @@ class GPU {
 	void screenshot(const std::string& name) { renderer->screenshot(name); }
 	void deinitGraphicsContext() { renderer->deinitGraphicsContext(); }
 
-#if defined(PANDA3DS_FRONTEND_SDL)
-	void initGraphicsContext(SDL_Window* window) { renderer->initGraphicsContext(window); }
-#elif defined(PANDA3DS_FRONTEND_QT)
-	void initGraphicsContext(GL::Context* context) { renderer->initGraphicsContext(context); }
-#endif
+	void initGraphicsContext(void* context) { renderer->initGraphicsContext(context); }
 
 	void fireDMA(u32 dest, u32 source, u32 size);
 	void reset();

include/emulator.hpp

@@ -106,12 +106,8 @@ class Emulator {
 	bool loadELF(const std::filesystem::path& path);
 	bool loadELF(std::ifstream& file);
 
-#ifdef PANDA3DS_FRONTEND_QT
-	// For passing the GL context from Qt to the renderer
-	void initGraphicsContext(GL::Context* glContext) { gpu.initGraphicsContext(nullptr); }
-#else
-	void initGraphicsContext(SDL_Window* window) { gpu.initGraphicsContext(window); }
-#endif
+	// For passing the SDL Window, GL context, etc from the frontend to the renderer
+	void initGraphicsContext(void* context) { gpu.initGraphicsContext(context); }
 
 	RomFS::DumpingResult dumpRomFS(const std::filesystem::path& path);
 	void setOutputSize(u32 width, u32 height) { gpu.setOutputSize(width, height); }

include/panda_qt/main_window.hpp

@@ -20,7 +20,7 @@
 #include "panda_qt/cpu_debugger.hpp"
 #include "panda_qt/dsp_debugger.hpp"
 #include "panda_qt/patch_window.hpp"
-#include "panda_qt/screen.hpp"
+#include "panda_qt/screen/screen.hpp"
 #include "panda_qt/shader_editor.hpp"
 #include "panda_qt/text_editor.hpp"
 #include "panda_qt/thread_debugger.hpp"
@@ -136,7 +136,7 @@ class MainWindow : public QMainWindow {
 	void loadKeybindings();
 	void saveKeybindings();
 
-	// Tracks whether we are using an OpenGL-backed renderer or a Vulkan-backed renderer
+	// Tracks what graphics API is backing our renderer
 	bool usingGL = false;
 	bool usingVk = false;
 	bool usingMtl = false;

include/panda_qt/screen/screen.hpp

@@ -1,25 +1,27 @@
 #pragma once
 #include <QWidget>
 #include <functional>
-#include <memory>
 
 #include "gl/context.h"
 #include "screen_layout.hpp"
 #include "window_info.h"
 
-// OpenGL widget for drawing the 3DS screen
+// Abstract screen widget for drawing the 3DS screen. We've got a child class for each graphics API (ScreenWidgetGL, ScreenWidgetMTL, ...)
 class ScreenWidget : public QWidget {
 	Q_OBJECT
 
   public:
 	using ResizeCallback = std::function<void(u32, u32)>;
 
-	ScreenWidget(ResizeCallback resizeCallback, QWidget* parent = nullptr);
-	void resizeEvent(QResizeEvent* event) override;
-	// Called by the emulator thread for resizing the actual GL surface, since the emulator thread owns the GL context
-	void resizeSurface(u32 width, u32 height);
+	enum class API { OpenGL, Metal, Vulkan };
 
-	GL::Context* getGLContext() { return glContext.get(); }
+	ScreenWidget(API api, ResizeCallback resizeCallback, QWidget* parent = nullptr);
+	virtual ~ScreenWidget() {}
+
+	void resizeEvent(QResizeEvent* event) override;
+
+	virtual GL::Context* getGLContext() { return nullptr; }
+	virtual void* getMTKLayer() { return nullptr; }
 
 	// Dimensions of our output surface
 	u32 surfaceWidth = 0;
@@ -30,8 +32,9 @@ class ScreenWidget : public QWidget {
 	u32 previousWidth = 0;
 	u32 previousHeight = 0;
 
-	// Coordinates (x/y/width/height) for the two screens in window space, used for properly handling touchscreen regardless
-	// of layout or resizing
+	API api = API::OpenGL;
+
+	// Coordinates (x/y/width/height) for the two screens in window space, used for properly handling touchscreen
 	ScreenLayout::WindowCoordinates screenCoordinates;
 	// Screen layouts and sizes
 	ScreenLayout::Layout screenLayout = ScreenLayout::Layout::Default;
@@ -39,16 +42,23 @@ class ScreenWidget : public QWidget {
 
 	void reloadScreenLayout(ScreenLayout::Layout newLayout, float newTopScreenSize);
 
-  private:
-	std::unique_ptr<GL::Context> glContext = nullptr;
+	// Creates a screen widget depending on the graphics API we're using
+	static ScreenWidget* getWidget(API api, ResizeCallback resizeCallback, QWidget* parent = nullptr);
+
+	// Called by the emulator thread on OpenGL for resizing the actual GL surface, since the emulator thread owns the GL context
+	virtual void resizeSurface(u32 width, u32 height) {};
+
+  protected:
 	ResizeCallback resizeCallback;
 
-	bool createGLContext();
+	virtual bool createContext() = 0;
+	virtual void resizeDisplay() = 0;
+	std::optional<WindowInfo> getWindowInfo();
 
+  private:
 	qreal devicePixelRatioFromScreen() const;
 	int scaledWindowWidth() const;
 	int scaledWindowHeight() const;
-	std::optional<WindowInfo> getWindowInfo();
 
 	void reloadScreenCoordinates();
 };

include/panda_qt/screen/screen_gl.hpp

@@ -0,0 +1,18 @@
+#pragma once
+#include <memory>
+
+#include "gl/context.h"
+#include "panda_qt/screen/screen.hpp"
+
+class ScreenWidgetGL : public ScreenWidget {
+	std::unique_ptr<GL::Context> glContext = nullptr;
+
+  public:
+	ScreenWidgetGL(API api, ResizeCallback resizeCallback, QWidget* parent = nullptr);
+
+	virtual GL::Context* getGLContext() override;
+	virtual bool createContext() override;
+
+	virtual void resizeDisplay() override;
+	virtual void resizeSurface(u32 width, u32 height) override;
+};

include/panda_qt/screen/screen_mtl.hpp

@@ -0,0 +1,18 @@
+#pragma once
+#include "panda_qt/screen/screen.hpp"
+
+class ScreenWidgetMTL : public ScreenWidget {
+	void* mtkLayer = nullptr;
+
+	// Objective-C++ functions for handling the Metal context
+	bool createMetalContext();
+	void resizeMetalView();
+
+  public:
+	ScreenWidgetMTL(API api, ResizeCallback resizeCallback, QWidget* parent = nullptr);
+	~ScreenWidgetMTL() override;
+
+	virtual void* getMTKLayer() override;
+	virtual bool createContext() override;
+	virtual void resizeDisplay() override;
+};

include/renderer.hpp

@@ -69,7 +69,7 @@ class Renderer {
 
 	virtual void reset() = 0;
 	virtual void display() = 0;                                                              // Display the 3DS screen contents to the window
-	virtual void initGraphicsContext(SDL_Window* window) = 0;                                // Initialize graphics context
+	virtual void initGraphicsContext(void* context) = 0;                                     // Initialize graphics context
 	virtual void clearBuffer(u32 startAddress, u32 endAddress, u32 value, u32 control) = 0;  // Clear a GPU buffer in VRAM
 	virtual void displayTransfer(u32 inputAddr, u32 outputAddr, u32 inputSize, u32 outputSize, u32 flags) = 0;  // Perform display transfer
 	virtual void textureCopy(u32 inputAddr, u32 outputAddr, u32 totalBytes, u32 inputSize, u32 outputSize, u32 flags) = 0;
@@ -91,9 +91,9 @@ class Renderer {
 	// Called to notify the core to use OpenGL ES and not desktop GL
 	virtual void setupGLES() {}
 
-	// Only relevant for Metal renderer on iOS
-	// Passes a SwiftUI MTKView's layer (CAMetalLayer) to the renderer
-	virtual void setMTKLayer(void* layer) {};
+	// Used for Metal renderer on Qt and iOS
+	// Passes an NSView's backing layer (CAMetalLayer) to the renderer
+	virtual void setMTKLayer(void* layer) { Helpers::panic("Renderer doesn't support MTK Layer"); };
 
 	// This function is called on every draw call before parsing vertex data.
 	// It is responsible for things like looking up which vertex/fragment shaders to use, recompiling them if they don't exist, choosing between
@@ -101,11 +101,6 @@ class Renderer {
 	// Returns whether this draw is eligible for using hardware-accelerated shaders or if shaders should run on the CPU
 	virtual bool prepareForDraw(ShaderUnit& shaderUnit, PICA::DrawAcceleration* accel) { return false; }
 
-	// Functions for initializing the graphics context for the Qt frontend, where we don't have the convenience of SDL_Window
-#ifdef PANDA3DS_FRONTEND_QT
-	virtual void initGraphicsContext(GL::Context* context) { Helpers::panic("Tried to initialize incompatible renderer with GL context"); }
-#endif
-
 	void setFBSize(u32 width, u32 height) {
 		fbSize[0] = width;
 		fbSize[1] = height;

include/renderer_gl/renderer_gl.hpp

@@ -191,7 +191,7 @@ class RendererGL final : public Renderer {
 
 	void reset() override;
 	void display() override;                                                              // Display the 3DS screen contents to the window
-	void initGraphicsContext(SDL_Window* window) override;                                // Initialize graphics context
+	void initGraphicsContext(void* context) override;                                     // Initialize graphics context
 	void clearBuffer(u32 startAddress, u32 endAddress, u32 value, u32 control) override;  // Clear a GPU buffer in VRAM
 	void displayTransfer(u32 inputAddr, u32 outputAddr, u32 inputSize, u32 outputSize, u32 flags) override;  // Perform display transfer
 	void textureCopy(u32 inputAddr, u32 outputAddr, u32 totalBytes, u32 inputSize, u32 outputSize, u32 flags) override;
@@ -211,10 +211,6 @@ class RendererGL final : public Renderer {
 	void resetStateManager() { gl.reset(); }
 	void initUbershader(OpenGL::Program& program);
 
-#ifdef PANDA3DS_FRONTEND_QT
-	virtual void initGraphicsContext([[maybe_unused]] GL::Context* context) override { initGraphicsContextInternal(); }
-#endif
-
 	// Take a screenshot of the screen and store it in a file
 	void screenshot(const std::string& name) override;
 };

include/renderer_mtl/renderer_mtl.hpp

@@ -13,7 +13,6 @@
 #include "mtl_vertex_buffer_cache.hpp"
 #include "renderer.hpp"
 
-
 // HACK: use the OpenGL cache
 #include "../renderer_gl/surface_cache.hpp"
 
@@ -30,7 +29,7 @@ class RendererMTL final : public Renderer {
 
 	void reset() override;
 	void display() override;
-	void initGraphicsContext(SDL_Window* window) override;
+	void initGraphicsContext(void* context) override;
 	void clearBuffer(u32 startAddress, u32 endAddress, u32 value, u32 control) override;
 	void displayTransfer(u32 inputAddr, u32 outputAddr, u32 inputSize, u32 outputSize, u32 flags) override;
 	void textureCopy(u32 inputAddr, u32 outputAddr, u32 totalBytes, u32 inputSize, u32 outputSize, u32 flags) override;
@@ -38,10 +37,6 @@ class RendererMTL final : public Renderer {
 	void screenshot(const std::string& name) override;
 	void deinitGraphicsContext() override;
 
-#ifdef PANDA3DS_FRONTEND_QT
-	virtual void initGraphicsContext([[maybe_unused]] GL::Context* context) override {}
-#endif
-
 	virtual void setMTKLayer(void* layer) override;
 
   private:

include/renderer_null/renderer_null.hpp

@@ -9,7 +9,7 @@ class RendererNull final : public Renderer {
 
 	void reset() override;
 	void display() override;
-	void initGraphicsContext(SDL_Window* window) override;
+	void initGraphicsContext(void* context) override;
 	void clearBuffer(u32 startAddress, u32 endAddress, u32 value, u32 control) override;
 	void displayTransfer(u32 inputAddr, u32 outputAddr, u32 inputSize, u32 outputSize, u32 flags) override;
 	void textureCopy(u32 inputAddr, u32 outputAddr, u32 totalBytes, u32 inputSize, u32 outputSize, u32 flags) override;
@@ -20,8 +20,4 @@ class RendererNull final : public Renderer {
 	// Tell the GPU core that we'll handle vertex fetch & shader execution in the renderer in order to speed up execution.
 	// Of course, we don't do this and geometry is never actually processed, since this is the null renderer.
 	virtual bool prepareForDraw(ShaderUnit& shaderUnit, PICA::DrawAcceleration* accel) override { return true; };
-
-#ifdef PANDA3DS_FRONTEND_QT
-	virtual void initGraphicsContext([[maybe_unused]] GL::Context* context) override {}
-#endif
 };

include/renderer_sw/renderer_sw.hpp

@@ -9,15 +9,11 @@ class RendererSw final : public Renderer {
 
 	void reset() override;
 	void display() override;
-	void initGraphicsContext(SDL_Window* window) override;
+	void initGraphicsContext(void* context) override;
 	void clearBuffer(u32 startAddress, u32 endAddress, u32 value, u32 control) override;
 	void displayTransfer(u32 inputAddr, u32 outputAddr, u32 inputSize, u32 outputSize, u32 flags) override;
 	void textureCopy(u32 inputAddr, u32 outputAddr, u32 totalBytes, u32 inputSize, u32 outputSize, u32 flags) override;
 	void drawVertices(PICA::PrimType primType, std::span<const PICA::Vertex> vertices) override;
 	void screenshot(const std::string& name) override;
 	void deinitGraphicsContext() override;
-
-#ifdef PANDA3DS_FRONTEND_QT
-	virtual void initGraphicsContext([[maybe_unused]] GL::Context* context) override {}
-#endif
 };

include/renderer_vk/renderer_vk.hpp

@@ -113,7 +113,7 @@ class RendererVK final : public Renderer {
 
 	void reset() override;
 	void display() override;
-	void initGraphicsContext(SDL_Window* window) override;
+	void initGraphicsContext(void* context) override;
 	void clearBuffer(u32 startAddress, u32 endAddress, u32 value, u32 control) override;
 	void displayTransfer(u32 inputAddr, u32 outputAddr, u32 inputSize, u32 outputSize, u32 flags) override;
 	void textureCopy(u32 inputAddr, u32 outputAddr, u32 totalBytes, u32 inputSize, u32 outputSize, u32 flags) override;