diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index 9e5346a1..ff4dc7f6 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -470,7 +470,7 @@ static void wlr_drm_connector_transform(struct wlr_output *output,
 
 static bool wlr_drm_connector_set_cursor(struct wlr_output *output,
 		const uint8_t *buf, int32_t stride, uint32_t width, uint32_t height,
-		int32_t hotspot_x, int32_t hotspot_y) {
+		int32_t hotspot_x, int32_t hotspot_y, bool update_pixels) {
 	struct wlr_drm_connector *conn = (struct wlr_drm_connector *)output;
 	struct wlr_drm_backend *drm = conn->drm;
 	struct wlr_drm_renderer *renderer = &drm->renderer;
@@ -478,7 +478,8 @@ static bool wlr_drm_connector_set_cursor(struct wlr_output *output,
 	struct wlr_drm_crtc *crtc = conn->crtc;
 	struct wlr_drm_plane *plane = crtc->cursor;
 
-	if (!buf) {
+	if (!buf && update_pixels) {
+		// Hide the cursor
 		return drm->iface->crtc_set_cursor(drm, crtc, NULL);
 	}
 
@@ -566,6 +567,11 @@ static bool wlr_drm_connector_set_cursor(struct wlr_output *output,
 		break;
 	}
 
+	if (!update_pixels) {
+		// Only update the cursor hotspot
+		return true;
+	}
+
 	struct gbm_bo *bo = plane->cursor_bo;
 	uint32_t bo_width = gbm_bo_get_width(bo);
 	uint32_t bo_height = gbm_bo_get_height(bo);
diff --git a/backend/wayland/output.c b/backend/wayland/output.c
index 03b43b35..2616b347 100644
--- a/backend/wayland/output.c
+++ b/backend/wayland/output.c
@@ -54,11 +54,18 @@ static void wlr_wl_output_transform(struct wlr_output *_output,
 
 static bool wlr_wl_output_set_cursor(struct wlr_output *_output,
 		const uint8_t *buf, int32_t stride, uint32_t width, uint32_t height,
-		int32_t hotspot_x, int32_t hotspot_y) {
+		int32_t hotspot_x, int32_t hotspot_y, bool update_pixels) {
 	struct wlr_wl_backend_output *output = (struct wlr_wl_backend_output *)_output;
 	struct wlr_wl_backend *backend = output->backend;
 
+	if (!update_pixels) {
+		// Update hotspot without changing cursor image
+		wlr_wl_output_update_cursor(output, output->enter_serial, hotspot_x,
+			hotspot_y);
+		return true;
+	}
 	if (!buf) {
+		// Hide cursor
 		wl_pointer_set_cursor(output->backend->pointer, output->enter_serial,
 			NULL, 0, 0);
 		return true;
diff --git a/include/wlr/interfaces/wlr_output.h b/include/wlr/interfaces/wlr_output.h
index 7d2821e0..b7927569 100644
--- a/include/wlr/interfaces/wlr_output.h
+++ b/include/wlr/interfaces/wlr_output.h
@@ -8,16 +8,16 @@ struct wlr_output_impl {
 	void (*enable)(struct wlr_output *output, bool enable);
 	bool (*set_mode)(struct wlr_output *output, struct wlr_output_mode *mode);
 	void (*transform)(struct wlr_output *output,
-			enum wl_output_transform transform);
+		enum wl_output_transform transform);
 	bool (*set_cursor)(struct wlr_output *output, const uint8_t *buf,
-			int32_t stride, uint32_t width, uint32_t height,
-			int32_t hotspot_x, int32_t hotspot_y);
+		int32_t stride, uint32_t width, uint32_t height,
+		int32_t hotspot_x, int32_t hotspot_y, bool update_pixels);
 	bool (*move_cursor)(struct wlr_output *output, int x, int y);
 	void (*destroy)(struct wlr_output *output);
 	void (*make_current)(struct wlr_output *output);
 	void (*swap_buffers)(struct wlr_output *output);
 	void (*set_gamma)(struct wlr_output *output,
-			uint16_t size, uint16_t *r, uint16_t *g, uint16_t *b);
+		uint16_t size, uint16_t *r, uint16_t *g, uint16_t *b);
 	uint16_t (*get_gamma_size)(struct wlr_output *output);
 };
 
@@ -25,6 +25,6 @@ void wlr_output_init(struct wlr_output *output, const struct wlr_output_impl *im
 void wlr_output_free(struct wlr_output *output);
 void wlr_output_update_matrix(struct wlr_output *output);
 struct wl_global *wlr_output_create_global(
-		struct wlr_output *wlr_output, struct wl_display *display);
+	struct wlr_output *wlr_output, struct wl_display *display);
 
 #endif
diff --git a/types/wlr_output.c b/types/wlr_output.c
index 0b3f3842..568ea093 100644
--- a/types/wlr_output.c
+++ b/types/wlr_output.c
@@ -127,7 +127,7 @@ static bool set_cursor(struct wlr_output *output, const uint8_t *buf,
 		int32_t hotspot_y) {
 	if (output->impl->set_cursor
 			&& output->impl->set_cursor(output, buf, stride, width, height,
-				hotspot_x, hotspot_y)) {
+				hotspot_x, hotspot_y, true)) {
 		output->cursor.is_sw = false;
 		return true;
 	}
@@ -238,6 +238,15 @@ void wlr_output_set_cursor_surface(struct wlr_output *output,
 	output->cursor.hotspot_x = hotspot_x;
 	output->cursor.hotspot_y = hotspot_y;
 
+	if (surface && surface == output->cursor.surface) {
+		if (output->impl->set_cursor && !output->cursor.is_sw) {
+			// Only update the hotspot
+			output->impl->set_cursor(output, NULL, 0, 0, 0, hotspot_x,
+				hotspot_y, false);
+		}
+		return;
+	}
+
 	if (output->cursor.surface) {
 		wl_list_remove(&output->cursor.surface_commit.link);
 		wl_list_remove(&output->cursor.surface_destroy.link);
diff --git a/types/wlr_surface.c b/types/wlr_surface.c
index 654b5f1c..a5760d67 100644
--- a/types/wlr_surface.c
+++ b/types/wlr_surface.c
@@ -427,8 +427,9 @@ static void wlr_surface_commit_pending(struct wlr_surface *surface) {
 	// TODO: add the invalid bitfield to this callback
 	wl_signal_emit(&surface->events.commit, surface);
 
-	// TODO: call this
-	//wlr_surface_state_release_buffer(surface->current);
+	// Release the buffer after calling commit, because some listeners
+	// might need it (e.g. for cursor surfaces)
+	wlr_surface_state_release_buffer(surface->current);
 }
 
 static bool wlr_subsurface_is_synchronized(struct wlr_subsurface *subsurface) {