diff --git a/include/wlr/types/wlr_seat.h b/include/wlr/types/wlr_seat.h
index cc7a5927..f5d5c357 100644
--- a/include/wlr/types/wlr_seat.h
+++ b/include/wlr/types/wlr_seat.h
@@ -75,11 +75,16 @@ struct wlr_seat_pointer_state {
 	struct wlr_seat_handle *focused_handle;
 	struct wlr_surface *focused_surface;
 
-	struct wl_listener surface_destroy;
-	struct wl_listener resource_destroy;
-
 	struct wlr_seat_pointer_grab *grab;
 	struct wlr_seat_pointer_grab *default_grab;
+
+	uint32_t button_count;
+	uint32_t grab_button;
+	uint32_t grab_serial;
+	uint32_t grab_time;
+
+	struct wl_listener surface_destroy;
+	struct wl_listener resource_destroy;
 };
 
 struct wlr_seat_keyboard {
diff --git a/types/wlr_seat.c b/types/wlr_seat.c
index 1285f2ee..bad0e5cf 100644
--- a/types/wlr_seat.c
+++ b/types/wlr_seat.c
@@ -538,8 +538,24 @@ void wlr_seat_pointer_notify_motion(struct wlr_seat *wlr_seat, uint32_t time,
 
 uint32_t wlr_seat_pointer_notify_button(struct wlr_seat *wlr_seat,
 		uint32_t time, uint32_t button, uint32_t state) {
+	if (state == WL_POINTER_BUTTON_STATE_PRESSED) {
+		if (wlr_seat->pointer_state.button_count == 0) {
+			wlr_seat->pointer_state.grab_button = button;
+			wlr_seat->pointer_state.grab_time = time;
+		}
+		wlr_seat->pointer_state.button_count++;
+	} else {
+		wlr_seat->pointer_state.button_count--;
+	}
+
 	struct wlr_seat_pointer_grab *grab = wlr_seat->pointer_state.grab;
-	return grab->interface->button(grab, time, button, state);
+	uint32_t serial = grab->interface->button(grab, time, button, state);
+
+	if (wlr_seat->pointer_state.button_count == 1) {
+		wlr_seat->pointer_state.grab_serial = serial;
+	}
+
+	return serial;
 }
 
 void wlr_seat_pointer_notify_axis(struct wlr_seat *wlr_seat, uint32_t time,