Return failure of wlr_renderer_init_wl_display()

This makes it easier for the user of this library to properly handle
failure of this function.

The signature of wlr_renderer_impl.init_wl_display was also modified to
allow for proper error propagation.

include/wlr/render/interface.h

@@ -62,7 +62,7 @@ struct wlr_renderer_impl {
 	struct wlr_texture *(*texture_from_dmabuf)(struct wlr_renderer *renderer,
 		struct wlr_dmabuf_attributes *attribs);
 	void (*destroy)(struct wlr_renderer *renderer);
-	void (*init_wl_display)(struct wlr_renderer *renderer,
+	bool (*init_wl_display)(struct wlr_renderer *renderer,
 		struct wl_display *wl_display);
 };
 

include/wlr/render/wlr_renderer.h

@@ -109,7 +109,12 @@ bool wlr_renderer_read_pixels(struct wlr_renderer *r, enum wl_shm_format fmt,
  */
 bool wlr_renderer_format_supported(struct wlr_renderer *r,
 	enum wl_shm_format fmt);
-void wlr_renderer_init_wl_display(struct wlr_renderer *r,
+/**
+ * Creates necessary shm and invokes the initialization of the implementation.
+ *
+ * Returns false on failure.
+ */
+bool wlr_renderer_init_wl_display(struct wlr_renderer *r,
 	struct wl_display *wl_display);
 
 /**

render/gles2/renderer.c

@@ -351,13 +351,15 @@ static struct wlr_texture *gles2_texture_from_dmabuf(
 	return wlr_gles2_texture_from_dmabuf(renderer->egl, attribs);
 }
 
-static void gles2_init_wl_display(struct wlr_renderer *wlr_renderer,
+static bool gles2_init_wl_display(struct wlr_renderer *wlr_renderer,
 		struct wl_display *wl_display) {
 	struct wlr_gles2_renderer *renderer =
 		gles2_get_renderer(wlr_renderer);
 	if (!wlr_egl_bind_display(renderer->egl, wl_display)) {
 		wlr_log(WLR_INFO, "failed to bind wl_display to EGL");
+		return false;
 	}
+	return true;
 }
 
 struct wlr_egl *wlr_gles2_renderer_get_egl(struct wlr_renderer *wlr_renderer) {

render/wlr_renderer.c

@@ -160,18 +160,18 @@ bool wlr_renderer_format_supported(struct wlr_renderer *r,
 	return r->impl->format_supported(r, fmt);
 }
 
-void wlr_renderer_init_wl_display(struct wlr_renderer *r,
+bool wlr_renderer_init_wl_display(struct wlr_renderer *r,
 		struct wl_display *wl_display) {
 	if (wl_display_init_shm(wl_display)) {
 		wlr_log(WLR_ERROR, "Failed to initialize shm");
-		return;
+		return false;
 	}
 
 	size_t len;
 	const enum wl_shm_format *formats = wlr_renderer_get_formats(r, &len);
 	if (formats == NULL) {
 		wlr_log(WLR_ERROR, "Failed to initialize shm: cannot get formats");
-		return;
+		return false;
 	}
 
 	for (size_t i = 0; i < len; ++i) {
@@ -183,14 +183,20 @@ void wlr_renderer_init_wl_display(struct wlr_renderer *r,
 	}
 
 	if (r->impl->texture_from_dmabuf) {
-		wlr_linux_dmabuf_v1_create(wl_display, r);
+		if (wlr_linux_dmabuf_v1_create(wl_display, r) == NULL) {
+			return false;
+		}
 	}
 
 	if (r->impl->init_wl_display) {
-		r->impl->init_wl_display(r, wl_display);
+		if (!r->impl->init_wl_display(r, wl_display)) {
+			return false;
 		}
 	}
 
+	return true;
+}
+
 struct wlr_renderer *wlr_renderer_autocreate(struct wlr_egl *egl,
 		EGLenum platform, void *remote_display, EGLint *config_attribs,
 		EGLint visual_id) {