output: Add function to set preferred render format

This change introduces new double buffered state to the wlr_output,
corresponding to the buffer format to render to.

The format being rendered to does not control the bit depth of colors
being sent to the display; it does generally determine the format with
which screenshot data is provided. The DRM backend _may_ sent higher
bit depths if the render format depth is increased, but hardware and
other limitations may apply.

include/wlr/interfaces/wlr_output.h

@@ -21,6 +21,7 @@
 	(WLR_OUTPUT_STATE_DAMAGE | \
 	WLR_OUTPUT_STATE_SCALE | \
 	WLR_OUTPUT_STATE_TRANSFORM | \
+	WLR_OUTPUT_STATE_RENDER_FORMAT | \
 	WLR_OUTPUT_STATE_ADAPTIVE_SYNC_ENABLED)
 
 /**

include/wlr/types/wlr_output.h

@@ -61,6 +61,7 @@ enum wlr_output_state_field {
 	WLR_OUTPUT_STATE_TRANSFORM = 1 << 5,
 	WLR_OUTPUT_STATE_ADAPTIVE_SYNC_ENABLED = 1 << 6,
 	WLR_OUTPUT_STATE_GAMMA_LUT = 1 << 7,
+	WLR_OUTPUT_STATE_RENDER_FORMAT = 1 << 8,
 };
 
 enum wlr_output_state_mode_type {
@@ -78,6 +79,7 @@ struct wlr_output_state {
 	float scale;
 	enum wl_output_transform transform;
 	bool adaptive_sync_enabled;
+	uint32_t render_format;
 
 	// only valid if WLR_OUTPUT_STATE_BUFFER
 	struct wlr_buffer *buffer;
@@ -134,6 +136,7 @@ struct wlr_output {
 	enum wl_output_subpixel subpixel;
 	enum wl_output_transform transform;
 	enum wlr_output_adaptive_sync_status adaptive_sync_status;
+	uint32_t render_format;
 
 	bool needs_frame;
 	// damage for cursors and fullscreen surface, in output-local coordinates
@@ -308,6 +311,22 @@ void wlr_output_set_transform(struct wlr_output *output,
  * Adaptive sync is double-buffered state, see `wlr_output_commit`.
  */
 void wlr_output_enable_adaptive_sync(struct wlr_output *output, bool enabled);
+/**
+ * Set the output buffer render format. Default value: DRM_FORMAT_XRGB8888
+ *
+ * While high bit depth render formats are necessary for a monitor to receive
+ * useful high bit data, they do not guarantee it; a DRM_FORMAT_XBGR2101010
+ * buffer will only lead to sending 10-bpc image data to the monitor if
+ * hardware and software permit this.
+ *
+ * This only affects the format of the output buffer used when rendering,
+ * as with `wlr_output_attach_render`. It has no impact on the cursor buffer
+ * format, or on the formats supported for direct scan-out (see also
+ * `wlr_output_attach_buffer`).
+ *
+ * This format is double-buffered state, see `wlr_output_commit`.
+ */
+void wlr_output_set_render_format(struct wlr_output *output, uint32_t format);
 /**
  * Sets a scale for the output.
  *

types/output/output.c

@@ -1,10 +1,13 @@
 #define _POSIX_C_SOURCE 200809L
 #include <assert.h>
+#include <backend/backend.h>
+#include <drm_fourcc.h>
 #include <stdlib.h>
 #include <wlr/interfaces/wlr_output.h>
 #include <wlr/types/wlr_matrix.h>
 #include <wlr/types/wlr_surface.h>
 #include <wlr/util/log.h>
+#include "render/allocator/allocator.h"
 #include "render/swapchain.h"
 #include "types/wlr_output.h"
 #include "util/global.h"
@@ -296,6 +299,16 @@ void wlr_output_enable_adaptive_sync(struct wlr_output *output, bool enabled) {
 	output->pending.adaptive_sync_enabled = enabled;
 }
 
+void wlr_output_set_render_format(struct wlr_output *output, uint32_t format) {
+	if (output->render_format == format) {
+		output->pending.committed &= ~WLR_OUTPUT_STATE_RENDER_FORMAT;
+		return;
+	}
+
+	output->pending.committed |= WLR_OUTPUT_STATE_RENDER_FORMAT;
+	output->pending.render_format = format;
+}
+
 void wlr_output_set_subpixel(struct wlr_output *output,
 		enum wl_output_subpixel subpixel) {
 	if (output->subpixel == subpixel) {
@@ -343,6 +356,7 @@ void wlr_output_init(struct wlr_output *output, struct wlr_backend *backend,
 	output->impl = impl;
 	output->display = display;
 	wl_list_init(&output->modes);
+	output->render_format = DRM_FORMAT_XRGB8888;
 	output->transform = WL_OUTPUT_TRANSFORM_NORMAL;
 	output->scale = 1;
 	output->commit_seq = 0;
@@ -542,6 +556,30 @@ static bool output_basic_test(struct wlr_output *output) {
 		}
 	}
 
+	if (output->pending.committed & WLR_OUTPUT_STATE_RENDER_FORMAT) {
+		struct wlr_allocator *allocator = output->allocator;
+		assert(allocator != NULL);
+
+		const struct wlr_drm_format_set *display_formats = NULL;
+		if (output->impl->get_primary_formats) {
+			display_formats =
+				output->impl->get_primary_formats(output, allocator->buffer_caps);
+			if (display_formats == NULL) {
+				wlr_log(WLR_ERROR, "Failed to get primary display formats");
+				return false;
+			}
+		}
+
+		struct wlr_drm_format *format = output_pick_format(output, display_formats,
+			output->pending.render_format);
+		if (format == NULL) {
+			wlr_log(WLR_ERROR, "Failed to pick primary buffer format for output");
+			return false;
+		}
+
+		free(format);
+	}
+
 	bool enabled = output->enabled;
 	if (output->pending.committed & WLR_OUTPUT_STATE_ENABLED) {
 		enabled = output->pending.enabled;
@@ -569,6 +607,10 @@ static bool output_basic_test(struct wlr_output *output) {
 		wlr_log(WLR_DEBUG, "Tried to enable adaptive sync on a disabled output");
 		return false;
 	}
+	if (!enabled && output->pending.committed & WLR_OUTPUT_STATE_RENDER_FORMAT) {
+		wlr_log(WLR_DEBUG, "Tried to set format for a disabled output");
+		return false;
+	}
 	if (!enabled && output->pending.committed & WLR_OUTPUT_STATE_GAMMA_LUT) {
 		wlr_log(WLR_DEBUG, "Tried to set the gamma lut on a disabled output");
 		return false;
@@ -642,6 +684,10 @@ bool wlr_output_commit(struct wlr_output *output) {
 		}
 	}
 
+	if (output->pending.committed & WLR_OUTPUT_STATE_RENDER_FORMAT) {
+		output->render_format = output->pending.render_format;
+	}
+
 	output->commit_seq++;
 
 	bool scale_updated = output->pending.committed & WLR_OUTPUT_STATE_SCALE;

types/output/render.c

@@ -9,6 +9,7 @@
 #include "render/drm_format_set.h"
 #include "render/swapchain.h"
 #include "render/wlr_renderer.h"
+#include "render/pixel_format.h"
 #include "types/wlr_output.h"
 
 bool wlr_output_init_render(struct wlr_output *output,
@@ -47,12 +48,6 @@ static bool output_create_swapchain(struct wlr_output *output,
 	int width, height;
 	output_pending_resolution(output, &width, &height);
 
-	if (output->swapchain != NULL && output->swapchain->width == width &&
-			output->swapchain->height == height &&
-			(allow_modifiers || output->swapchain->format->len == 0)) {
-		return true;
-	}
-
 	struct wlr_allocator *allocator = output->allocator;
 	assert(allocator != NULL);
 
@@ -67,12 +62,22 @@ static bool output_create_swapchain(struct wlr_output *output,
 	}
 
 	struct wlr_drm_format *format = output_pick_format(output, display_formats,
-		DRM_FORMAT_XRGB8888);
+		output->render_format);
 	if (format == NULL) {
 		wlr_log(WLR_ERROR, "Failed to pick primary buffer format for output '%s'",
 			output->name);
 		return false;
 	}
+
+	if (output->swapchain != NULL && output->swapchain->width == width &&
+			output->swapchain->height == height &&
+			output->swapchain->format->format == format->format &&
+			(allow_modifiers || output->swapchain->format->len == 0)) {
+		// no change, keep existing swapchain
+		free(format);
+		return true;
+	}
+
 	wlr_log(WLR_DEBUG, "Choosing primary buffer format 0x%"PRIX32" for output '%s'",
 		format->format, output->name);
 
@@ -171,6 +176,9 @@ bool output_ensure_buffer(struct wlr_output *output) {
 	if (output->pending.committed & WLR_OUTPUT_STATE_MODE) {
 		needs_new_buffer = true;
 	}
+	if (output->pending.committed & WLR_OUTPUT_STATE_RENDER_FORMAT) {
+		needs_new_buffer = true;
+	}
 	if (!needs_new_buffer ||
 			(output->pending.committed & WLR_OUTPUT_STATE_BUFFER)) {
 		return true;