diff --git a/backend/drm/backend.c b/backend/drm/backend.c
index 87a4f708..6be5e5d5 100644
--- a/backend/drm/backend.c
+++ b/backend/drm/backend.c
@@ -34,6 +34,10 @@ static void wlr_drm_backend_destroy(struct wlr_backend *backend) {
 		wlr_output_destroy(&conn->output);
 	}
 
+	wl_list_remove(&drm->display_destroy.link);
+	wl_list_remove(&drm->session_signal.link);
+	wl_list_remove(&drm->drm_invalidated.link);
+
 	wlr_drm_resources_free(drm);
 	wlr_drm_renderer_finish(&drm->renderer);
 	wlr_session_close_file(drm->session, drm->fd);
@@ -57,7 +61,8 @@ bool wlr_backend_is_drm(struct wlr_backend *b) {
 }
 
 static void session_signal(struct wl_listener *listener, void *data) {
-	struct wlr_drm_backend *drm = wl_container_of(listener, drm, session_signal);
+	struct wlr_drm_backend *drm =
+		wl_container_of(listener, drm, session_signal);
 	struct wlr_session *session = data;
 
 	if (session->active) {
@@ -84,7 +89,8 @@ static void session_signal(struct wl_listener *listener, void *data) {
 }
 
 static void drm_invalidated(struct wl_listener *listener, void *data) {
-	struct wlr_drm_backend *drm = wl_container_of(listener, drm, drm_invalidated);
+	struct wlr_drm_backend *drm =
+		wl_container_of(listener, drm, drm_invalidated);
 
 	char *name = drmGetDeviceNameFromFd2(drm->fd);
 	wlr_log(L_DEBUG, "%s invalidated", name);
@@ -93,6 +99,12 @@ static void drm_invalidated(struct wl_listener *listener, void *data) {
 	wlr_drm_scan_connectors(drm);
 }
 
+static void handle_display_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_drm_backend *drm =
+		wl_container_of(listener, drm, display_destroy);
+	wlr_drm_backend_destroy(&drm->backend);
+}
+
 struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 		struct wlr_session *session, int gpu_fd, struct wlr_backend *parent) {
 	assert(display && session && gpu_fd >= 0);
@@ -150,6 +162,9 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 		wlr_log(L_INFO, "Failed to bind egl/wl display: %s", egl_error());
 	}
 
+	drm->display_destroy.notify = handle_display_destroy;
+	wl_display_add_destroy_listener(display, &drm->display_destroy);
+
 	return &drm->backend;
 
 error_event:
diff --git a/backend/libinput/backend.c b/backend/libinput/backend.c
index 1028e238..c9352051 100644
--- a/backend/libinput/backend.c
+++ b/backend/libinput/backend.c
@@ -43,10 +43,12 @@ static void wlr_libinput_log(struct libinput *libinput_context,
 }
 
 static bool wlr_libinput_backend_start(struct wlr_backend *_backend) {
-	struct wlr_libinput_backend *backend = (struct wlr_libinput_backend *)_backend;
+	struct wlr_libinput_backend *backend =
+		(struct wlr_libinput_backend *)_backend;
 	wlr_log(L_DEBUG, "Initializing libinput");
-	backend->libinput_context = libinput_udev_create_context(&libinput_impl, backend,
-			backend->session->udev);
+
+	backend->libinput_context = libinput_udev_create_context(&libinput_impl,
+		backend, backend->session->udev);
 	if (!backend->libinput_context) {
 		wlr_log(L_ERROR, "Failed to create libinput context");
 		return false;
@@ -99,6 +101,7 @@ static void wlr_libinput_backend_destroy(struct wlr_backend *_backend) {
 	}
 	struct wlr_libinput_backend *backend =
 		(struct wlr_libinput_backend *)_backend;
+
 	for (size_t i = 0; i < backend->wlr_device_lists.length; i++) {
 		struct wl_list *wlr_devices = backend->wlr_device_lists.items[i];
 		struct wlr_input_device *wlr_dev, *next;
@@ -108,6 +111,10 @@ static void wlr_libinput_backend_destroy(struct wlr_backend *_backend) {
 		}
 		free(wlr_devices);
 	}
+
+	wl_list_remove(&backend->display_destroy.link);
+	wl_list_remove(&backend->session_signal.link);
+
 	wlr_list_finish(&backend->wlr_device_lists);
 	wl_event_source_remove(backend->input_event);
 	libinput_unref(backend->libinput_context);
@@ -124,7 +131,8 @@ bool wlr_backend_is_libinput(struct wlr_backend *b) {
 }
 
 static void session_signal(struct wl_listener *listener, void *data) {
-	struct wlr_libinput_backend *backend = wl_container_of(listener, backend, session_signal);
+	struct wlr_libinput_backend *backend =
+		wl_container_of(listener, backend, session_signal);
 	struct wlr_session *session = data;
 
 	if (!backend->libinput_context) {
@@ -138,6 +146,12 @@ static void session_signal(struct wl_listener *listener, void *data) {
 	}
 }
 
+static void handle_display_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_libinput_backend *backend =
+		wl_container_of(listener, backend, display_destroy);
+	wlr_libinput_backend_destroy(&backend->backend);
+}
+
 struct wlr_backend *wlr_libinput_backend_create(struct wl_display *display,
 		struct wlr_session *session) {
 	assert(display && session);
@@ -160,6 +174,9 @@ struct wlr_backend *wlr_libinput_backend_create(struct wl_display *display,
 	backend->session_signal.notify = session_signal;
 	wl_signal_add(&session->session_signal, &backend->session_signal);
 
+	backend->display_destroy.notify = handle_display_destroy;
+	wl_display_add_destroy_listener(display, &backend->display_destroy);
+
 	return &backend->backend;
 error_backend:
 	free(backend);
diff --git a/backend/wayland/backend.c b/backend/wayland/backend.c
index 1801f3e0..cb241490 100644
--- a/backend/wayland/backend.c
+++ b/backend/wayland/backend.c
@@ -78,6 +78,8 @@ static void wlr_wl_backend_destroy(struct wlr_backend *_backend) {
 		wlr_input_device_destroy(input_device);
 	}
 
+	wl_list_remove(&backend->local_display_destroy.link);
+
 	free(backend->seat_name);
 
 	wl_event_source_remove(backend->remote_display_src);
@@ -117,6 +119,12 @@ struct wlr_wl_backend_output *wlr_wl_output_for_surface(
 	return NULL;
 }
 
+static void handle_display_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_wl_backend *backend =
+		wl_container_of(listener, backend, local_display_destroy);
+	wlr_wl_backend_destroy(&backend->backend);
+}
+
 struct wlr_backend *wlr_wl_backend_create(struct wl_display *display) {
 	wlr_log(L_INFO, "Creating wayland backend");
 
@@ -143,8 +151,12 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display) {
 		return false;
 	}
 
-	wlr_egl_init(&backend->egl, EGL_PLATFORM_WAYLAND_EXT, WL_SHM_FORMAT_ARGB8888, backend->remote_display);
+	wlr_egl_init(&backend->egl, EGL_PLATFORM_WAYLAND_EXT,
+		WL_SHM_FORMAT_ARGB8888, backend->remote_display);
 	wlr_egl_bind_display(&backend->egl, backend->local_display);
 
+	backend->local_display_destroy.notify = handle_display_destroy;
+	wl_display_add_destroy_listener(display, &backend->local_display_destroy);
+
 	return &backend->backend;
 }
diff --git a/backend/x11/backend.c b/backend/x11/backend.c
index b798daf6..a4ca55b0 100644
--- a/backend/x11/backend.c
+++ b/backend/x11/backend.c
@@ -184,69 +184,6 @@ static void init_atom(struct wlr_x11_backend *x11, struct wlr_x11_atom *atom,
 	atom->reply = xcb_intern_atom_reply(x11->xcb_conn, atom->cookie, NULL);
 }
 
-struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
-		const char *x11_display) {
-	struct wlr_x11_backend *x11 = calloc(1, sizeof(*x11));
-	if (!x11) {
-		return NULL;
-	}
-
-	wlr_backend_init(&x11->backend, &backend_impl);
-	x11->wl_display = display;
-
-	x11->xlib_conn = XOpenDisplay(x11_display);
-	if (!x11->xlib_conn) {
-		wlr_log(L_ERROR, "Failed to open X connection");
-		return NULL;
-	}
-
-	x11->xcb_conn = XGetXCBConnection(x11->xlib_conn);
-	if (!x11->xcb_conn || xcb_connection_has_error(x11->xcb_conn)) {
-		wlr_log(L_ERROR, "Failed to open xcb connection");
-		goto error_x11;
-	}
-
-	XSetEventQueueOwner(x11->xlib_conn, XCBOwnsEventQueue);
-
-	int fd = xcb_get_file_descriptor(x11->xcb_conn);
-	struct wl_event_loop *ev = wl_display_get_event_loop(display);
-	int events = WL_EVENT_READABLE | WL_EVENT_ERROR | WL_EVENT_HANGUP;
-	x11->event_source = wl_event_loop_add_fd(ev, fd, events, x11_event, x11);
-	if (!x11->event_source) {
-		wlr_log(L_ERROR, "Could not create event source");
-		goto error_x11;
-	}
-
-	x11->frame_timer = wl_event_loop_add_timer(ev, signal_frame, x11);
-
-	x11->screen = xcb_setup_roots_iterator(xcb_get_setup(x11->xcb_conn)).data;
-
-	if (!wlr_egl_init(&x11->egl, EGL_PLATFORM_X11_KHR,
-			x11->screen->root_visual, x11->xlib_conn)) {
-		goto error_event;
-	}
-
-	wlr_input_device_init(&x11->keyboard_dev, WLR_INPUT_DEVICE_KEYBOARD,
-		NULL, "X11 keyboard", 0, 0);
-	wlr_keyboard_init(&x11->keyboard, NULL);
-	x11->keyboard_dev.keyboard = &x11->keyboard;
-
-	wlr_input_device_init(&x11->pointer_dev, WLR_INPUT_DEVICE_POINTER,
-		NULL, "X11 pointer", 0, 0);
-	wlr_pointer_init(&x11->pointer, NULL);
-	x11->pointer_dev.pointer = &x11->pointer;
-
-	return &x11->backend;
-
-error_event:
-	wl_event_source_remove(x11->event_source);
-error_x11:
-	xcb_disconnect(x11->xcb_conn);
-	XCloseDisplay(x11->xlib_conn);
-	free(x11);
-	return NULL;
-}
-
 static bool wlr_x11_backend_start(struct wlr_backend *backend) {
 	struct wlr_x11_backend *x11 = (struct wlr_x11_backend *)backend;
 	struct wlr_x11_output *output = &x11->output;
@@ -307,6 +244,8 @@ static void wlr_x11_backend_destroy(struct wlr_backend *backend) {
 	struct wlr_x11_output *output = &x11->output;
 	wlr_output_destroy(&output->wlr_output);
 
+	wl_list_remove(&x11->display_destroy.link);
+
 	wl_event_source_remove(x11->frame_timer);
 	wlr_egl_free(&x11->egl);
 
@@ -329,6 +268,78 @@ static struct wlr_backend_impl backend_impl = {
 	.get_egl = wlr_x11_backend_get_egl,
 };
 
+static void handle_display_destroy(struct wl_listener *listener, void *data) {
+	struct wlr_x11_backend *x11 =
+		wl_container_of(listener, x11, display_destroy);
+	wlr_x11_backend_destroy(&x11->backend);
+}
+
+struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
+		const char *x11_display) {
+	struct wlr_x11_backend *x11 = calloc(1, sizeof(*x11));
+	if (!x11) {
+		return NULL;
+	}
+
+	wlr_backend_init(&x11->backend, &backend_impl);
+	x11->wl_display = display;
+
+	x11->xlib_conn = XOpenDisplay(x11_display);
+	if (!x11->xlib_conn) {
+		wlr_log(L_ERROR, "Failed to open X connection");
+		return NULL;
+	}
+
+	x11->xcb_conn = XGetXCBConnection(x11->xlib_conn);
+	if (!x11->xcb_conn || xcb_connection_has_error(x11->xcb_conn)) {
+		wlr_log(L_ERROR, "Failed to open xcb connection");
+		goto error_x11;
+	}
+
+	XSetEventQueueOwner(x11->xlib_conn, XCBOwnsEventQueue);
+
+	int fd = xcb_get_file_descriptor(x11->xcb_conn);
+	struct wl_event_loop *ev = wl_display_get_event_loop(display);
+	int events = WL_EVENT_READABLE | WL_EVENT_ERROR | WL_EVENT_HANGUP;
+	x11->event_source = wl_event_loop_add_fd(ev, fd, events, x11_event, x11);
+	if (!x11->event_source) {
+		wlr_log(L_ERROR, "Could not create event source");
+		goto error_x11;
+	}
+
+	x11->frame_timer = wl_event_loop_add_timer(ev, signal_frame, x11);
+
+	x11->screen = xcb_setup_roots_iterator(xcb_get_setup(x11->xcb_conn)).data;
+
+	if (!wlr_egl_init(&x11->egl, EGL_PLATFORM_X11_KHR,
+			x11->screen->root_visual, x11->xlib_conn)) {
+		goto error_event;
+	}
+
+	wlr_input_device_init(&x11->keyboard_dev, WLR_INPUT_DEVICE_KEYBOARD,
+		NULL, "X11 keyboard", 0, 0);
+	wlr_keyboard_init(&x11->keyboard, NULL);
+	x11->keyboard_dev.keyboard = &x11->keyboard;
+
+	wlr_input_device_init(&x11->pointer_dev, WLR_INPUT_DEVICE_POINTER,
+		NULL, "X11 pointer", 0, 0);
+	wlr_pointer_init(&x11->pointer, NULL);
+	x11->pointer_dev.pointer = &x11->pointer;
+
+	x11->display_destroy.notify = handle_display_destroy;
+	wl_display_add_destroy_listener(display, &x11->display_destroy);
+
+	return &x11->backend;
+
+error_event:
+	wl_event_source_remove(x11->event_source);
+error_x11:
+	xcb_disconnect(x11->xcb_conn);
+	XCloseDisplay(x11->xlib_conn);
+	free(x11);
+	return NULL;
+}
+
 static void output_transform(struct wlr_output *wlr_output, enum wl_output_transform transform) {
 	struct wlr_x11_output *output = (struct wlr_x11_output *)wlr_output;
 	output->wlr_output.transform = transform;
diff --git a/include/backend/drm/drm.h b/include/backend/drm/drm.h
index 4fad9678..1f70a103 100644
--- a/include/backend/drm/drm.h
+++ b/include/backend/drm/drm.h
@@ -91,6 +91,7 @@ struct wlr_drm_backend {
 	struct wl_display *display;
 	struct wl_event_source *drm_event;
 
+	struct wl_listener display_destroy;
 	struct wl_listener session_signal;
 	struct wl_listener drm_invalidated;
 
diff --git a/include/backend/libinput.h b/include/backend/libinput.h
index be2557ed..6ec718c6 100644
--- a/include/backend/libinput.h
+++ b/include/backend/libinput.h
@@ -16,6 +16,7 @@ struct wlr_libinput_backend {
 	struct libinput *libinput_context;
 	struct wl_event_source *input_event;
 
+	struct wl_listener display_destroy;
 	struct wl_listener session_signal;
 
 	struct wlr_list wlr_device_lists; // list of struct wl_list
diff --git a/include/backend/wayland.h b/include/backend/wayland.h
index 1e8a55d2..713c291c 100644
--- a/include/backend/wayland.h
+++ b/include/backend/wayland.h
@@ -21,6 +21,7 @@ struct wlr_wl_backend {
 	struct wl_list outputs;
 	struct wlr_egl egl;
 	size_t requested_outputs;
+	struct wl_listener local_display_destroy;
 	/* remote state */
 	struct wl_display *remote_display;
 	struct wl_event_source *remote_display_src;
diff --git a/include/backend/x11.h b/include/backend/x11.h
index f5ec56bc..31a5138b 100644
--- a/include/backend/x11.h
+++ b/include/backend/x11.h
@@ -51,6 +51,8 @@ struct wlr_x11_backend {
 
 	// The time we last received an event
 	xcb_timestamp_t time;
+
+	struct wl_listener display_destroy;
 };
 
 #endif