diff --git a/backend/x11/backend.c b/backend/x11/backend.c
index 79b896fe..7d560d97 100644
--- a/backend/x11/backend.c
+++ b/backend/x11/backend.c
@@ -7,6 +7,7 @@
 #include <wayland-server.h>
 #include <wlr/backend/interface.h>
 #include <wlr/backend/x11.h>
+#include <wlr/config.h>
 #include <wlr/interfaces/wlr_input_device.h>
 #include <wlr/interfaces/wlr_keyboard.h>
 #include <wlr/interfaces/wlr_output.h>
@@ -16,12 +17,14 @@
 #include <wlr/util/log.h>
 #include <X11/Xlib-xcb.h>
 #include <xcb/xcb.h>
-#include <xcb/xkb.h>
 #ifdef __linux__
 #include <linux/input-event-codes.h>
 #elif __FreeBSD__
 #include <dev/evdev/input-event-codes.h>
 #endif
+#ifdef WLR_HAS_XCB_XKB
+#include <xcb/xkb.h>
+#endif
 #include "backend/x11.h"
 #include "util/signal.h"
 
@@ -31,10 +34,6 @@ static const struct wlr_backend_impl backend_impl;
 static const struct wlr_output_impl output_impl;
 static const struct wlr_input_device_impl input_device_impl = { 0 };
 
-// TODO: remove global state
-static uint8_t xkb_base_event;
-static uint8_t xkb_base_error;
-
 static uint32_t xcb_button_to_wl(uint32_t button) {
 	switch (button) {
 	case XCB_BUTTON_INDEX_1: return BTN_LEFT;
@@ -162,12 +161,14 @@ static bool handle_x11_event(struct wlr_x11_backend *x11, xcb_generic_event_t *e
 		break;
 	}
 	default:
-		if (event->response_type == xkb_base_event) {
+#ifdef WLR_HAS_XCB_XKB
+		if (x11->xkb_supported && event->response_type == x11->xkb_base_event) {
 			xcb_xkb_state_notify_event_t *ev =
 				(xcb_xkb_state_notify_event_t *)event;
 			wlr_keyboard_notify_modifiers(&x11->keyboard, ev->baseMods,
 				ev->latchedMods, ev->lockedMods, ev->lockedGroup);
 		}
+#endif
 		break;
 	}
 
@@ -236,28 +237,6 @@ static bool wlr_x11_backend_start(struct wlr_backend *backend) {
 		x11->screen->root, 0, 0, 1024, 768, 1, XCB_WINDOW_CLASS_INPUT_OUTPUT,
 		x11->screen->root_visual, mask, values);
 
-	const xcb_query_extension_reply_t *reply =
-		xcb_get_extension_data(x11->xcb_conn, &xcb_xkb_id);
-	if (reply->present) {
-		xkb_base_event = reply->first_event;
-		xkb_base_error = reply->first_error;
-
-		xcb_xkb_use_extension_cookie_t cookie = xcb_xkb_use_extension(
-			x11->xcb_conn, XCB_XKB_MAJOR_VERSION, XCB_XKB_MINOR_VERSION);
-		xcb_xkb_use_extension_reply_t *reply =
-			xcb_xkb_use_extension_reply(x11->xcb_conn, cookie, NULL);
-		if (reply && reply->supported) {
-			xcb_xkb_select_events(x11->xcb_conn,
-				XCB_XKB_ID_USE_CORE_KBD,
-				XCB_XKB_EVENT_TYPE_STATE_NOTIFY,
-				0,
-				XCB_XKB_EVENT_TYPE_STATE_NOTIFY,
-				0,
-				0,
-				0);
-		}
-	}
-
 	output->surf = wlr_egl_create_surface(&x11->egl, &output->win);
 	if (!output->surf) {
 		wlr_log(L_ERROR, "Failed to create EGL surface");
@@ -315,6 +294,32 @@ static bool wlr_x11_backend_start(struct wlr_backend *backend) {
 			strlen(title), title);
 	}
 
+#ifdef WLR_HAS_XCB_XKB
+		const xcb_query_extension_reply_t *reply =
+			xcb_get_extension_data(x11->xcb_conn, &xcb_xkb_id);
+		if (reply != NULL && reply->present) {
+			x11->xkb_base_event = reply->first_event;
+			x11->xkb_base_error = reply->first_error;
+
+			xcb_xkb_use_extension_cookie_t cookie = xcb_xkb_use_extension(
+				x11->xcb_conn, XCB_XKB_MAJOR_VERSION, XCB_XKB_MINOR_VERSION);
+			xcb_xkb_use_extension_reply_t *reply =
+				xcb_xkb_use_extension_reply(x11->xcb_conn, cookie, NULL);
+			if (reply != NULL && reply->supported) {
+				x11->xkb_supported = true;
+
+				xcb_xkb_select_events(x11->xcb_conn,
+					XCB_XKB_ID_USE_CORE_KBD,
+					XCB_XKB_EVENT_TYPE_STATE_NOTIFY,
+					0,
+					XCB_XKB_EVENT_TYPE_STATE_NOTIFY,
+					0,
+					0,
+					0);
+			}
+		}
+#endif
+
 	xcb_map_window(x11->xcb_conn, output->win);
 	xcb_flush(x11->xcb_conn);
 	wlr_output_update_enabled(&output->wlr_output, true);
diff --git a/include/backend/x11.h b/include/backend/x11.h
index 840509bf..72710f6c 100644
--- a/include/backend/x11.h
+++ b/include/backend/x11.h
@@ -48,6 +48,12 @@ struct wlr_x11_backend {
 	// The time we last received an event
 	xcb_timestamp_t time;
 
+#ifdef WLR_HAS_XCB_XKB
+	bool xkb_supported;
+	uint8_t xkb_base_event;
+	uint8_t xkb_base_error;
+#endif
+
 	struct wl_listener display_destroy;
 };
 
diff --git a/meson.build b/meson.build
index a48984c1..04c93817 100644
--- a/meson.build
+++ b/meson.build
@@ -85,18 +85,22 @@ if get_option('enable_x11_backend') or get_option('enable_xwayland')
 	xcb            = dependency('xcb')
 	xcb_composite  = dependency('xcb-composite')
 	xcb_xfixes     = dependency('xcb-xfixes')
-	xcb_xkb        = dependency('xcb-xkb') # TODO: make this optional
 	xcb_image      = dependency('xcb-image')
 	xcb_render     = dependency('xcb-render')
 	x11_xcb        = dependency('x11-xcb')
 
 	xcb_icccm      = dependency('xcb-icccm', required: false)
+	xcb_xkb        = dependency('xcb-xkb', required: false)
 	xcb_errors     = dependency('xcb-errors', required: get_option('enable_xcb_errors') == 'true')
 
 	if xcb_icccm.found()
 		conf_data.set('WLR_HAS_XCB_ICCCM', true)
 	endif
 
+	if xcb_xkb.found()
+		conf_data.set('WLR_HAS_XCB_XKB', true)
+	endif
+
 	if xcb_errors.found() and get_option('enable_xcb_errors') != 'false'
 		conf_data.set('WLR_HAS_XCB_ERRORS', true)
 	endif