output: use DRM format in wlr_output_preferred_read_format

include/wlr/render/interface.h

@@ -42,7 +42,7 @@ struct wlr_renderer_impl {
 		struct wlr_renderer *renderer);
 	const struct wlr_drm_format_set *(*get_dmabuf_render_formats)(
 		struct wlr_renderer *renderer);
-	enum wl_shm_format (*preferred_read_format)(struct wlr_renderer *renderer);
+	uint32_t (*preferred_read_format)(struct wlr_renderer *renderer);
 	bool (*read_pixels)(struct wlr_renderer *renderer, enum wl_shm_format fmt,
 		uint32_t *flags, uint32_t stride, uint32_t width, uint32_t height,
 		uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,

include/wlr/types/wlr_output.h

@@ -340,8 +340,7 @@ void wlr_output_attach_buffer(struct wlr_output *output,
  * Get the preferred format for reading pixels.
  * This function might change the current rendering context.
  */
-bool wlr_output_preferred_read_format(struct wlr_output *output,
-	enum wl_shm_format *fmt);
+uint32_t wlr_output_preferred_read_format(struct wlr_output *output);
 /**
  * Set the damage region for the frame to be submitted. This is the region of
  * the screen that has changed since the last frame.

render/gles2/renderer.c

@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <GLES2/gl2.h>
 #include <GLES2/gl2ext.h>
 #include <stdint.h>
@@ -434,7 +435,7 @@ static const struct wlr_drm_format_set *gles2_get_dmabuf_render_formats(
 	return wlr_egl_get_dmabuf_render_formats(renderer->egl);
 }
 
-static enum wl_shm_format gles2_preferred_read_format(
+static uint32_t gles2_preferred_read_format(
 		struct wlr_renderer *wlr_renderer) {
 	struct wlr_gles2_renderer *renderer =
 		gles2_get_renderer_in_context(wlr_renderer);
@@ -456,13 +457,13 @@ static enum wl_shm_format gles2_preferred_read_format(
 	const struct wlr_gles2_pixel_format *fmt =
 		get_gles2_format_from_gl(gl_format, gl_type, alpha_size > 0);
 	if (fmt != NULL) {
-		return convert_drm_format_to_wl_shm(fmt->drm_format);
+		return fmt->drm_format;
 	}
 
 	if (renderer->exts.read_format_bgra_ext) {
-		return WL_SHM_FORMAT_XRGB8888;
+		return DRM_FORMAT_XRGB8888;
 	}
-	return WL_SHM_FORMAT_XBGR8888;
+	return DRM_FORMAT_XBGR8888;
 }
 
 static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,

types/wlr_output.c

@@ -1,5 +1,6 @@
 #define _POSIX_C_SOURCE 200809L
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <stdlib.h>
 #include <string.h>
 #include <tgmath.h>
@@ -450,19 +451,18 @@ bool wlr_output_attach_render(struct wlr_output *output, int *buffer_age) {
 	return true;
 }
 
-bool wlr_output_preferred_read_format(struct wlr_output *output,
-		enum wl_shm_format *fmt) {
+uint32_t wlr_output_preferred_read_format(struct wlr_output *output) {
 	struct wlr_renderer *renderer = wlr_backend_get_renderer(output->backend);
 	if (!renderer->impl->preferred_read_format || !renderer->impl->read_pixels) {
-		return false;
+		return DRM_FORMAT_INVALID;
 	}
 
 	if (!output->impl->attach_render(output, NULL)) {
-		return false;
+		return DRM_FORMAT_INVALID;
 	}
-	*fmt = renderer->impl->preferred_read_format(renderer);
+	uint32_t fmt = renderer->impl->preferred_read_format(renderer);
 	output->impl->rollback_render(output);
-	return true;
+	return fmt;
 }
 
 void wlr_output_set_damage(struct wlr_output *output,

types/wlr_screencopy_v1.c

@@ -8,6 +8,7 @@
 #include <wlr/backend.h>
 #include <wlr/util/log.h>
 #include "wlr-screencopy-unstable-v1-protocol.h"
+#include "render/shm_format.h"
 #include "util/signal.h"
 
 #define SCREENCOPY_MANAGER_VERSION 3
@@ -551,12 +552,14 @@ static void capture_output(struct wl_client *wl_client,
 	struct wlr_renderer *renderer = wlr_backend_get_renderer(output->backend);
 	assert(renderer);
 
-	if (!wlr_output_preferred_read_format(frame->output, &frame->format)) {
+	uint32_t drm_format = wlr_output_preferred_read_format(frame->output);
+	if (drm_format == DRM_FORMAT_INVALID) {
 		wlr_log(WLR_ERROR,
 			"Failed to capture output: no read format supported by renderer");
 		goto error;
 	}
 
+	frame->format = convert_drm_format_to_wl_shm(drm_format);
 	frame->fourcc = get_output_fourcc(output);
 
 	struct wlr_box buffer_box = {0};