From ea77cc5cb2a8ba287b1c737771d17028dc7cfcc3 Mon Sep 17 00:00:00 2001
From: Thomas Hebb <tommyhebb@gmail.com>
Date: Wed, 5 Jan 2022 00:16:59 -0800
Subject: [PATCH] render/gles2: don't constrain shm formats to ones that
 support reading

commit 44e8451cd93e ("render/gles2: hide shm formats without GL
support") added the is_gles2_pixel_format_supported() function to
render/gles2/pixel_format.c, whose stated purpose is to "check whether
the renderer has the needed GL extensions to read a given pixel format."
It then used that function to filter the pixel formats returned by
get_gles2_shm_formats().

The result of this change is that RGB formats are no longer reported for
GL drivers that don't implement EXT_read_format_bgra, even when those
formats are supported for rendering (which they have to be for
wlr_gles2_renderer_create() to succeed). This is a pretty clear
regression, since wlr_renderer_init_wl_shm() fails when either of
WL_SHM_FORMAT_ARGB8888 or WL_SHM_FORMAT_XRGB8888 are missing.

To fix the regression, change is_gles2_pixel_format_supported() to
accept all pixel formats that support rendering, regardless of whether
we can read them or not, and move the check for EXT_read_format_bgra
back into gles2_read_pixels(). (There's already a check for this
extension in gles2_preferred_read_format(), so we're not breaking any
abstraction that wasn't already broken.)

Tested on the NVIDIA 495.46 proprietary driver, which doesn't support
EXT_read_format_bgra.

Fixes: 44e8451cd93e ("render/gles2: hide shm formats without GL support")
(cherry picked from commit 59b9518f072527ac59593e51df7f5d5331a34f0e)
---
 render/gles2/pixel_format.c | 14 ++++++++++----
 render/gles2/renderer.c     |  6 ++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/render/gles2/pixel_format.c b/render/gles2/pixel_format.c
index 31bb3908..b155bbbe 100644
--- a/render/gles2/pixel_format.c
+++ b/render/gles2/pixel_format.c
@@ -98,6 +98,10 @@ static const struct wlr_gles2_pixel_format formats[] = {
 
 // TODO: more pixel formats
 
+/*
+ * Return true if supported for texturing, even if other operations like
+ * reading aren't supported.
+ */
 bool is_gles2_pixel_format_supported(const struct wlr_gles2_renderer *renderer,
 		const struct wlr_gles2_pixel_format *format) {
 	if (format->gl_type == GL_UNSIGNED_INT_2_10_10_10_REV_EXT
@@ -108,10 +112,12 @@ bool is_gles2_pixel_format_supported(const struct wlr_gles2_renderer *renderer,
 			&& !renderer->exts.OES_texture_half_float_linear) {
 		return false;
 	}
-	if (format->gl_format == GL_BGRA_EXT
-			&& !renderer->exts.EXT_read_format_bgra) {
-		return false;
-	}
+	/*
+	 * Note that we don't need to check for GL_EXT_texture_format_BGRA8888
+	 * here, since we've already checked if we have it at renderer creation
+	 * time and bailed out if not. We do the check there because Wayland
+	 * requires all compositors to support SHM buffers in that format.
+	 */
 	return true;
 }
 
diff --git a/render/gles2/renderer.c b/render/gles2/renderer.c
index 527d85bf..67b8ead4 100644
--- a/render/gles2/renderer.c
+++ b/render/gles2/renderer.c
@@ -441,6 +441,12 @@ static bool gles2_read_pixels(struct wlr_renderer *wlr_renderer,
 		return false;
 	}
 
+	if (fmt->gl_format == GL_BGRA_EXT && !renderer->exts.EXT_read_format_bgra) {
+		wlr_log(WLR_ERROR,
+			"Cannot read pixels: missing GL_EXT_read_format_bgra extension");
+		return false;
+	}
+
 	const struct wlr_pixel_format_info *drm_fmt =
 		drm_get_pixel_format_info(fmt->drm_format);
 	assert(drm_fmt);