From 7508f87fcbfa8c9feb72fcd38f323113059c1b83 Mon Sep 17 00:00:00 2001
From: Manuel Stoeckl <code@mstoeckl.com>
Date: Wed, 10 Nov 2021 22:54:04 -0500
Subject: [PATCH] output: lift up output format fallback logic

This makes it possible for the two functions using output_pick_format
(output_pick_cursor_format and output_create_swapchain) to select
different buffer formats.
---
 include/types/wlr_output.h |  2 +-
 types/output/cursor.c      |  8 ++++-
 types/output/render.c      | 66 ++++++++++++++++++--------------------
 3 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/include/types/wlr_output.h b/include/types/wlr_output.h
index cde64e3a..1159d00a 100644
--- a/include/types/wlr_output.h
+++ b/include/types/wlr_output.h
@@ -8,7 +8,7 @@ void output_pending_resolution(struct wlr_output *output, int *width,
 	int *height);
 
 struct wlr_drm_format *output_pick_format(struct wlr_output *output,
-	const struct wlr_drm_format_set *display_formats);
+	const struct wlr_drm_format_set *display_formats, uint32_t format);
 void output_clear_back_buffer(struct wlr_output *output);
 bool output_ensure_buffer(struct wlr_output *output);
 
diff --git a/types/output/cursor.c b/types/output/cursor.c
index f1e6aee5..e1414201 100644
--- a/types/output/cursor.c
+++ b/types/output/cursor.c
@@ -206,7 +206,13 @@ static struct wlr_drm_format *output_pick_cursor_format(struct wlr_output *outpu
 		}
 	}
 
-	return output_pick_format(output, display_formats);
+	struct wlr_drm_format *format = output_pick_format(output, display_formats,
+		DRM_FORMAT_ARGB8888);
+	if (format == NULL) {
+		format = output_pick_format(output, display_formats,
+			DRM_FORMAT_XRGB8888);
+	}
+	return format;
 }
 
 static struct wlr_buffer *render_cursor_buffer(struct wlr_output_cursor *cursor) {
diff --git a/types/output/render.c b/types/output/render.c
index 5bf5530e..8b35355d 100644
--- a/types/output/render.c
+++ b/types/output/render.c
@@ -66,7 +66,12 @@ static bool output_create_swapchain(struct wlr_output *output,
 		}
 	}
 
-	struct wlr_drm_format *format = output_pick_format(output, display_formats);
+	struct wlr_drm_format *format = output_pick_format(output, display_formats,
+		DRM_FORMAT_ARGB8888);
+	if (format == NULL) {
+		format = output_pick_format(output, display_formats,
+			DRM_FORMAT_XRGB8888);
+	}
 	if (format == NULL) {
 		wlr_log(WLR_ERROR, "Failed to pick primary buffer format for output '%s'",
 			output->name);
@@ -231,7 +236,8 @@ void wlr_output_lock_attach_render(struct wlr_output *output, bool lock) {
 }
 
 struct wlr_drm_format *output_pick_format(struct wlr_output *output,
-		const struct wlr_drm_format_set *display_formats) {
+		const struct wlr_drm_format_set *display_formats,
+		uint32_t fmt) {
 	struct wlr_renderer *renderer = output->renderer;
 	struct wlr_allocator *allocator = output->allocator;
 	assert(renderer != NULL && allocator != NULL);
@@ -243,41 +249,31 @@ struct wlr_drm_format *output_pick_format(struct wlr_output *output,
 		return NULL;
 	}
 
-	struct wlr_drm_format *format = NULL;
-	const uint32_t candidates[] = { DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB8888 };
-	for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); i++) {
-		uint32_t fmt = candidates[i];
-
-		const struct wlr_drm_format *render_format =
-			wlr_drm_format_set_get(render_formats, fmt);
-		if (render_format == NULL) {
-			wlr_log(WLR_DEBUG, "Renderer doesn't support format 0x%"PRIX32, fmt);
-			continue;
-		}
-
-		if (display_formats != NULL) {
-			const struct wlr_drm_format *display_format =
-				wlr_drm_format_set_get(display_formats, fmt);
-			if (display_format == NULL) {
-				wlr_log(WLR_DEBUG, "Output doesn't support format 0x%"PRIX32, fmt);
-				continue;
-			}
-			format = wlr_drm_format_intersect(display_format, render_format);
-		} else {
-			// The output can display any format
-			format = wlr_drm_format_dup(render_format);
-		}
-
-		if (format == NULL) {
-			wlr_log(WLR_DEBUG, "Failed to intersect display and render "
-				"modifiers for format 0x%"PRIX32, fmt);
-		} else {
-			break;
-		}
+	const struct wlr_drm_format *render_format =
+		wlr_drm_format_set_get(render_formats, fmt);
+	if (render_format == NULL) {
+		wlr_log(WLR_DEBUG, "Renderer doesn't support format 0x%"PRIX32, fmt);
+		return NULL;
 	}
+
+	struct wlr_drm_format *format = NULL;
+	if (display_formats != NULL) {
+		const struct wlr_drm_format *display_format =
+			wlr_drm_format_set_get(display_formats, fmt);
+		if (display_format == NULL) {
+			wlr_log(WLR_DEBUG, "Output doesn't support format 0x%"PRIX32, fmt);
+			return NULL;
+		}
+		format = wlr_drm_format_intersect(display_format, render_format);
+	} else {
+		// The output can display any format
+		format = wlr_drm_format_dup(render_format);
+	}
+
 	if (format == NULL) {
-		wlr_log(WLR_ERROR, "Failed to choose a format for output '%s'",
-			output->name);
+		wlr_log(WLR_DEBUG, "Failed to intersect display and render "
+			"modifiers for format 0x%"PRIX32 " on output '%s",
+			fmt, output->name);
 		return NULL;
 	}