diff --git a/include/wlr/types/wlr_buffer.h b/include/wlr/types/wlr_buffer.h
deleted file mode 100644
index fc348a1c..00000000
--- a/include/wlr/types/wlr_buffer.h
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef WLR_TYPES_WLR_BUFFER_H
-#define WLR_TYPES_WLR_BUFFER_H
-
-#include <pixman.h>
-#include <wayland-server.h>
-
-/**
- * A client buffer.
- */
-struct wlr_buffer {
-	struct wl_resource *resource; // can be NULL
-	struct wlr_texture *texture; // can be NULL
-	bool released;
-	size_t n_refs;
-
-	struct wl_listener resource_destroy;
-};
-
-struct wlr_renderer;
-
-/**
- * Check if a resource is a wl_buffer resource.
- */
-bool wlr_resource_is_buffer(struct wl_resource *resource);
-/**
- * Get the size of a wl_buffer resource.
- */
-bool wlr_buffer_get_resource_size(struct wl_resource *resource,
-	struct wlr_renderer *renderer, int *width, int *height);
-
-/**
- * Upload a buffer to the GPU and reference it.
- */
-struct wlr_buffer *wlr_buffer_create(struct wlr_renderer *renderer,
-	struct wl_resource *resource);
-/**
- * Reference the buffer.
- */
-struct wlr_buffer *wlr_buffer_ref(struct wlr_buffer *buffer);
-/**
- * Unreference the buffer. After this call, `buffer` may not be accessed
- * anymore.
- */
-void wlr_buffer_unref(struct wlr_buffer *buffer);
-/**
- * Try to update the buffer's content. On success, returns the updated buffer
- * and destroys the provided `buffer`. On error, `buffer` is intact and NULL is
- * returned.
- *
- * Fails if there's more than one reference to the buffer or if the texture
- * isn't mutable.
- */
-struct wlr_buffer *wlr_buffer_apply_damage(struct wlr_buffer *buffer,
-	struct wl_resource *resource, pixman_region32_t *damage);
-
-#endif
diff --git a/include/wlr/types/wlr_surface.h b/include/wlr/types/wlr_surface.h
index 64503e78..526e4e2c 100644
--- a/include/wlr/types/wlr_surface.h
+++ b/include/wlr/types/wlr_surface.h
@@ -69,7 +69,6 @@ struct wlr_subsurface {
 struct wlr_surface {
 	struct wl_resource *resource;
 	struct wlr_renderer *renderer;
-	struct wlr_buffer *buffer;
 	struct wlr_texture *texture;
 	struct wlr_surface_state *current, *pending;
 	const char *role; // the lifetime-bound role or null
diff --git a/types/meson.build b/types/meson.build
index 87f21c55..f9f5b469 100644
--- a/types/meson.build
+++ b/types/meson.build
@@ -20,7 +20,6 @@ lib_wlr_types = static_library(
 		'xdg_shell_v6/wlr_xdg_surface_v6.c',
 		'xdg_shell_v6/wlr_xdg_toplevel_v6.c',
 		'wlr_box.c',
-		'wlr_buffer.c',
 		'wlr_compositor.c',
 		'wlr_cursor.c',
 		'wlr_gamma_control.c',
diff --git a/types/wlr_buffer.c b/types/wlr_buffer.c
deleted file mode 100644
index 82a359f0..00000000
--- a/types/wlr_buffer.c
+++ /dev/null
@@ -1,187 +0,0 @@
-#include <assert.h>
-#include <stdlib.h>
-#include <wlr/types/wlr_buffer.h>
-#include <wlr/types/wlr_linux_dmabuf.h>
-#include <wlr/render/wlr_renderer.h>
-#include <wlr/util/log.h>
-
-bool wlr_resource_is_buffer(struct wl_resource *resource) {
-	return strcmp(wl_resource_get_class(resource), wl_buffer_interface.name) == 0;
-}
-
-bool wlr_buffer_get_resource_size(struct wl_resource *resource,
-		struct wlr_renderer *renderer, int *width, int *height) {
-	assert(wlr_resource_is_buffer(resource));
-
-	struct wl_shm_buffer *shm_buf = wl_shm_buffer_get(resource);
-	if (shm_buf != NULL) {
-		*width = wl_shm_buffer_get_width(shm_buf);
-		*height = wl_shm_buffer_get_height(shm_buf);
-	} else if (wlr_renderer_resource_is_wl_drm_buffer(renderer,
-			resource)) {
-		wlr_renderer_wl_drm_buffer_get_size(renderer, resource,
-			width, height);
-	} else if (wlr_dmabuf_resource_is_buffer(resource)) {
-		struct wlr_dmabuf_buffer *dmabuf =
-			wlr_dmabuf_buffer_from_buffer_resource(resource);
-		*width = dmabuf->attributes.width;
-		*height = dmabuf->attributes.height;
-	} else {
-		*width = *height = 0;
-		return false;
-	}
-
-	return true;
-}
-
-
-static void buffer_resource_handle_destroy(struct wl_listener *listener,
-		void *data) {
-	struct wlr_buffer *buffer =
-		wl_container_of(listener, buffer, resource_destroy);
-	wl_list_remove(&buffer->resource_destroy.link);
-	wl_list_init(&buffer->resource_destroy.link);
-	buffer->resource = NULL;
-
-	if (!buffer->released) {
-		// The texture becomes invalid
-		wlr_texture_destroy(buffer->texture);
-		buffer->texture = NULL;
-	}
-}
-
-struct wlr_buffer *wlr_buffer_create(struct wlr_renderer *renderer,
-		struct wl_resource *resource) {
-	assert(wlr_resource_is_buffer(resource));
-
-	struct wlr_texture *texture = NULL;
-	bool released = false;
-
-	struct wl_shm_buffer *shm_buf = wl_shm_buffer_get(resource);
-	if (shm_buf != NULL) {
-		enum wl_shm_format fmt = wl_shm_buffer_get_format(shm_buf);
-		int32_t stride = wl_shm_buffer_get_stride(shm_buf);
-		int32_t width = wl_shm_buffer_get_width(shm_buf);
-		int32_t height = wl_shm_buffer_get_height(shm_buf);
-
-		wl_shm_buffer_begin_access(shm_buf);
-		void *data = wl_shm_buffer_get_data(shm_buf);
-		texture = wlr_texture_from_pixels(renderer, fmt, stride,
-			width, height, data);
-		wl_shm_buffer_end_access(shm_buf);
-
-		// We have uploaded the data, we don't need to access the wl_buffer
-		// anymore
-		wl_buffer_send_release(resource);
-		released = true;
-	} else if (wlr_renderer_resource_is_wl_drm_buffer(renderer, resource)) {
-		texture = wlr_texture_from_wl_drm(renderer, resource);
-	} else if (wlr_dmabuf_resource_is_buffer(resource)) {
-		struct wlr_dmabuf_buffer *dmabuf =
-			wlr_dmabuf_buffer_from_buffer_resource(resource);
-		texture = wlr_texture_from_dmabuf(renderer, &dmabuf->attributes);
-	} else {
-		wlr_log(L_ERROR, "Cannot upload texture: unknown buffer type");
-		return NULL;
-	}
-
-	if (texture == NULL) {
-		wlr_log(L_ERROR, "Failed to upload texture");
-		return NULL;
-	}
-
-	struct wlr_buffer *buffer = calloc(1, sizeof(struct wlr_buffer));
-	if (buffer == NULL) {
-		return NULL;
-	}
-	buffer->resource = resource;
-	buffer->texture = texture;
-	buffer->released = released;
-	buffer->n_refs = 1;
-
-	wl_resource_add_destroy_listener(resource, &buffer->resource_destroy);
-	buffer->resource_destroy.notify = buffer_resource_handle_destroy;
-
-	return buffer;
-}
-
-struct wlr_buffer *wlr_buffer_ref(struct wlr_buffer *buffer) {
-	buffer->n_refs++;
-	return buffer;
-}
-
-void wlr_buffer_unref(struct wlr_buffer *buffer) {
-	if (buffer == NULL) {
-		return;
-	}
-
-	assert(buffer->n_refs > 0);
-	buffer->n_refs--;
-	if (buffer->n_refs > 0) {
-		return;
-	}
-
-	if (!buffer->released && buffer->resource != NULL) {
-		wl_buffer_send_release(buffer->resource);
-	}
-
-	wl_list_remove(&buffer->resource_destroy.link);
-	wlr_texture_destroy(buffer->texture);
-	free(buffer);
-}
-
-struct wlr_buffer *wlr_buffer_apply_damage(struct wlr_buffer *buffer,
-		struct wl_resource *resource, pixman_region32_t *damage) {
-	assert(wlr_resource_is_buffer(resource));
-
-	if (buffer->n_refs > 1) {
-		// Someone else still has a reference to the buffer
-		return NULL;
-	}
-
-	struct wl_shm_buffer *shm_buf = wl_shm_buffer_get(resource);
-	if (shm_buf == NULL) {
-		// Uploading only damaged regions only works for wl_shm buffers
-		return NULL;
-	}
-
-	enum wl_shm_format fmt = wl_shm_buffer_get_format(shm_buf);
-	int32_t stride = wl_shm_buffer_get_stride(shm_buf);
-	int32_t width = wl_shm_buffer_get_width(shm_buf);
-	int32_t height = wl_shm_buffer_get_height(shm_buf);
-
-	int32_t texture_width, texture_height;
-	wlr_texture_get_size(buffer->texture, &texture_width, &texture_height);
-	if (width != texture_width || height != texture_height) {
-		return NULL;
-	}
-
-	wl_shm_buffer_begin_access(shm_buf);
-	void *data = wl_shm_buffer_get_data(shm_buf);
-
-	int n;
-	pixman_box32_t *rects = pixman_region32_rectangles(damage, &n);
-	for (int i = 0; i < n; ++i) {
-		pixman_box32_t *r = &rects[i];
-		if (!wlr_texture_write_pixels(buffer->texture, fmt, stride,
-				r->x2 - r->x1, r->y2 - r->y1, r->x1, r->y1,
-				r->x1, r->y1, data)) {
-			wl_shm_buffer_end_access(shm_buf);
-			return NULL;
-		}
-	}
-
-	wl_shm_buffer_end_access(shm_buf);
-
-	// We have uploaded the data, we don't need to access the wl_buffer
-	// anymore
-	wl_buffer_send_release(resource);
-
-	wl_list_remove(&buffer->resource_destroy.link);
-	wl_resource_add_destroy_listener(resource, &buffer->resource_destroy);
-	buffer->resource_destroy.notify = buffer_resource_handle_destroy;
-
-	buffer->resource = resource;
-	buffer->released = true;
-	return buffer;
-}
diff --git a/types/wlr_surface.c b/types/wlr_surface.c
index 7d8da02f..fca4e847 100644
--- a/types/wlr_surface.c
+++ b/types/wlr_surface.c
@@ -1,9 +1,10 @@
 #include <assert.h>
 #include <stdlib.h>
 #include <wayland-server.h>
+#include <wlr/render/egl.h>
 #include <wlr/render/interface.h>
-#include <wlr/types/wlr_buffer.h>
 #include <wlr/types/wlr_compositor.h>
+#include <wlr/types/wlr_linux_dmabuf.h>
 #include <wlr/types/wlr_matrix.h>
 #include <wlr/types/wlr_region.h>
 #include <wlr/types/wlr_surface.h>
@@ -45,6 +46,13 @@ static void surface_handle_buffer_destroy(struct wl_listener *listener,
 	surface_state_reset_buffer(state);
 }
 
+static void surface_state_release_buffer(struct wlr_surface_state *state) {
+	if (state->buffer) {
+		wl_buffer_send_release(state->buffer);
+		surface_state_reset_buffer(state);
+	}
+}
+
 static void surface_state_set_buffer(struct wlr_surface_state *state,
 		struct wl_resource *buffer) {
 	surface_state_reset_buffer(state);
@@ -167,8 +175,24 @@ static bool surface_update_size(struct wlr_surface *surface,
 	int scale = state->scale;
 	enum wl_output_transform transform = state->transform;
 
-	wlr_buffer_get_resource_size(state->buffer, surface->renderer,
-		&state->buffer_width, &state->buffer_height);
+	struct wl_shm_buffer *buf = wl_shm_buffer_get(state->buffer);
+	if (buf != NULL) {
+		state->buffer_width = wl_shm_buffer_get_width(buf);
+		state->buffer_height = wl_shm_buffer_get_height(buf);
+	} else if (wlr_renderer_resource_is_wl_drm_buffer(surface->renderer,
+			state->buffer)) {
+		wlr_renderer_wl_drm_buffer_get_size(surface->renderer, state->buffer,
+			&state->buffer_width, &state->buffer_height);
+	} else if (wlr_dmabuf_resource_is_buffer(state->buffer)) {
+		struct wlr_dmabuf_buffer *dmabuf =
+			wlr_dmabuf_buffer_from_buffer_resource(state->buffer);
+		state->buffer_width = dmabuf->attributes.width;
+		state->buffer_height = dmabuf->attributes.height;
+	} else {
+		wlr_log(L_ERROR, "Unknown buffer handle attached");
+		state->buffer_width = 0;
+		state->buffer_height = 0;
+	}
 
 	int width = state->buffer_width / scale;
 	int height = state->buffer_height / scale;
@@ -219,6 +243,7 @@ static void surface_move_state(struct wlr_surface *surface,
 		update_size = true;
 	}
 	if ((next->invalid & WLR_SURFACE_INVALID_BUFFER)) {
+		surface_state_release_buffer(state);
 		surface_state_set_buffer(state, next->buffer);
 		surface_state_reset_buffer(next);
 		state->sx = next->sx;
@@ -326,60 +351,91 @@ static void surface_damage_subsurfaces(struct wlr_subsurface *subsurface) {
 }
 
 static void surface_apply_damage(struct wlr_surface *surface,
-		bool invalid_buffer) {
+		bool invalid_buffer, bool reupload_buffer) {
 	struct wl_resource *resource = surface->current->buffer;
 	if (resource == NULL) {
-		// NULL commit
-		wlr_buffer_unref(surface->buffer);
-		surface->buffer = NULL;
-		surface->texture = NULL;
 		return;
 	}
 
-	if (surface->buffer != NULL && !surface->buffer->released &&
-			!invalid_buffer) {
-		// The buffer is still the same, no need to re-upload
-		return;
-	}
+	struct wl_shm_buffer *buf = wl_shm_buffer_get(resource);
+	if (buf != NULL) {
+		wl_shm_buffer_begin_access(buf);
 
-	if (surface->buffer != NULL && surface->buffer->released) {
-		pixman_region32_t damage;
-		pixman_region32_init(&damage);
-		pixman_region32_copy(&damage, &surface->current->buffer_damage);
-		pixman_region32_intersect_rect(&damage, &damage, 0, 0,
-			surface->current->buffer_width, surface->current->buffer_height);
+		enum wl_shm_format fmt = wl_shm_buffer_get_format(buf);
+		int32_t stride = wl_shm_buffer_get_stride(buf);
+		int32_t width = wl_shm_buffer_get_width(buf);
+		int32_t height = wl_shm_buffer_get_height(buf);
+		void *data = wl_shm_buffer_get_data(buf);
 
-		struct wlr_buffer *updated_buffer =
-			wlr_buffer_apply_damage(surface->buffer, resource, &damage);
+		if (surface->texture == NULL || reupload_buffer) {
+			wlr_texture_destroy(surface->texture);
+			surface->texture = wlr_texture_from_pixels(surface->renderer, fmt,
+				stride, width, height, data);
+		} else {
+			pixman_region32_t damage;
+			pixman_region32_init(&damage);
+			pixman_region32_copy(&damage, &surface->current->buffer_damage);
+			pixman_region32_intersect_rect(&damage, &damage, 0, 0,
+				surface->current->buffer_width,
+				surface->current->buffer_height);
 
-		pixman_region32_fini(&damage);
+			int n;
+			pixman_box32_t *rects = pixman_region32_rectangles(&damage, &n);
+			for (int i = 0; i < n; ++i) {
+				pixman_box32_t *r = &rects[i];
+				if (!wlr_texture_write_pixels(surface->texture, fmt, stride,
+						r->x2 - r->x1, r->y2 - r->y1, r->x1, r->y1,
+						r->x1, r->y1, data)) {
+					break;
+				}
+			}
 
-		if (updated_buffer != NULL) {
-			surface->buffer = updated_buffer;
-			return;
+			pixman_region32_fini(&damage);
 		}
+
+		wl_shm_buffer_end_access(buf);
+
+		// We've uploaded the wl_shm_buffer data to the GPU, we won't access the
+		// wl_buffer anymore
+		surface_state_release_buffer(surface->current);
+	} else if (invalid_buffer || reupload_buffer) {
+		wlr_texture_destroy(surface->texture);
+
+		if (wlr_renderer_resource_is_wl_drm_buffer(surface->renderer, resource)) {
+			surface->texture =
+				wlr_texture_from_wl_drm(surface->renderer, resource);
+		} else if (wlr_dmabuf_resource_is_buffer(resource)) {
+			struct wlr_dmabuf_buffer *dmabuf =
+				wlr_dmabuf_buffer_from_buffer_resource(resource);
+			surface->texture =
+				wlr_texture_from_dmabuf(surface->renderer, &dmabuf->attributes);
+		} else {
+			surface->texture = NULL;
+			wlr_log(L_ERROR, "Unknown buffer handle attached");
+		}
+
+		// Don't release the wl_buffer yet: since the texture is shared with the
+		// client, we'll access the wl_buffer when rendering
 	}
-
-	wlr_buffer_unref(surface->buffer);
-	surface->buffer = NULL;
-	surface->texture = NULL;
-
-	struct wlr_buffer *buffer = wlr_buffer_create(surface->renderer, resource);
-	if (buffer == NULL) {
-		wlr_log(L_ERROR, "Failed to upload buffer");
-		return;
-	}
-
-	surface->buffer = buffer;
-	surface->texture = buffer->texture;
 }
 
 static void surface_commit_pending(struct wlr_surface *surface) {
+	int32_t oldw = surface->current->buffer_width;
+	int32_t oldh = surface->current->buffer_height;
+
 	bool invalid_buffer = surface->pending->invalid & WLR_SURFACE_INVALID_BUFFER;
+	bool null_buffer_commit = invalid_buffer && surface->pending->buffer == NULL;
 
 	surface_move_state(surface, surface->pending, surface->current);
 
-	surface_apply_damage(surface, invalid_buffer);
+	if (null_buffer_commit) {
+		wlr_texture_destroy(surface->texture);
+		surface->texture = NULL;
+	}
+
+	bool reupload_buffer = oldw != surface->current->buffer_width ||
+		oldh != surface->current->buffer_height;
+	surface_apply_damage(surface, invalid_buffer, reupload_buffer);
 
 	// commit subsurface order
 	struct wlr_subsurface *subsurface;
@@ -601,9 +657,10 @@ static void surface_handle_resource_destroy(struct wl_resource *resource) {
 	wl_list_remove(wl_resource_get_link(surface->resource));
 
 	wl_list_remove(&surface->renderer_destroy.link);
+	wlr_texture_destroy(surface->texture);
 	surface_state_destroy(surface->pending);
 	surface_state_destroy(surface->current);
-	wlr_buffer_unref(surface->buffer);
+
 	free(surface);
 }