123 lines
3.6 KiB
Diff
123 lines
3.6 KiB
Diff
From: Rusty Russell <rusty@rustcorp.com.au>
|
|
Date: Thu, 12 Jan 2012 15:44:43 +1030
|
|
Subject: virtio: support unlocked queue kick
|
|
|
|
commit 41f0377f73039ca6fe97a469d1941a89cd9757f1 upstream.
|
|
|
|
Based on patch by Christoph for virtio_blk speedup:
|
|
|
|
Split virtqueue_kick to be able to do the actual notification
|
|
outside the lock protecting the virtqueue. This patch was
|
|
originally done by Stefan Hajnoczi, but I can't find the
|
|
original one anymore and had to recreated it from memory.
|
|
Pointers to the original or corrections for the commit message
|
|
are welcome.
|
|
|
|
Stefan's patch was here:
|
|
|
|
https://github.com/stefanha/linux/commit/a6d06644e3a58e57a774e77d7dc34c4a5a2e7496
|
|
http://www.spinics.net/lists/linux-virtualization/msg14616.html
|
|
|
|
Third time's the charm!
|
|
|
|
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
|
|
[bwh: Backported to 3.2: adjust context]
|
|
---
|
|
drivers/virtio/virtio_ring.c | 60 +++++++++++++++++++++++++++++++++---------
|
|
include/linux/virtio.h | 4 +++
|
|
2 files changed, 52 insertions(+), 12 deletions(-)
|
|
|
|
--- a/drivers/virtio/virtio_ring.c
|
|
+++ b/drivers/virtio/virtio_ring.c
|
|
@@ -245,10 +245,23 @@ add_head:
|
|
}
|
|
EXPORT_SYMBOL_GPL(virtqueue_add_buf_gfp);
|
|
|
|
-void virtqueue_kick(struct virtqueue *_vq)
|
|
+/**
|
|
+ * virtqueue_kick_prepare - first half of split virtqueue_kick call.
|
|
+ * @vq: the struct virtqueue
|
|
+ *
|
|
+ * Instead of virtqueue_kick(), you can do:
|
|
+ * if (virtqueue_kick_prepare(vq))
|
|
+ * virtqueue_notify(vq);
|
|
+ *
|
|
+ * This is sometimes useful because the virtqueue_kick_prepare() needs
|
|
+ * to be serialized, but the actual virtqueue_notify() call does not.
|
|
+ */
|
|
+bool virtqueue_kick_prepare(struct virtqueue *_vq)
|
|
{
|
|
struct vring_virtqueue *vq = to_vvq(_vq);
|
|
u16 new, old;
|
|
+ bool needs_kick;
|
|
+
|
|
START_USE(vq);
|
|
/* Descriptors and available array need to be set before we expose the
|
|
* new available array entries. */
|
|
@@ -261,13 +274,46 @@ void virtqueue_kick(struct virtqueue *_v
|
|
/* Need to update avail index before checking if we should notify */
|
|
virtio_mb();
|
|
|
|
- if (vq->event ?
|
|
- vring_need_event(vring_avail_event(&vq->vring), new, old) :
|
|
- !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
|
|
- /* Prod other side to tell it about changes. */
|
|
- vq->notify(&vq->vq);
|
|
-
|
|
+ if (vq->event) {
|
|
+ needs_kick = vring_need_event(vring_avail_event(&vq->vring),
|
|
+ new, old);
|
|
+ } else {
|
|
+ needs_kick = !(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY);
|
|
+ }
|
|
END_USE(vq);
|
|
+ return needs_kick;
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
|
|
+
|
|
+/**
|
|
+ * virtqueue_notify - second half of split virtqueue_kick call.
|
|
+ * @vq: the struct virtqueue
|
|
+ *
|
|
+ * This does not need to be serialized.
|
|
+ */
|
|
+void virtqueue_notify(struct virtqueue *_vq)
|
|
+{
|
|
+ struct vring_virtqueue *vq = to_vvq(_vq);
|
|
+
|
|
+ /* Prod other side to tell it about changes. */
|
|
+ vq->notify(_vq);
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(virtqueue_notify);
|
|
+
|
|
+/**
|
|
+ * virtqueue_kick - update after add_buf
|
|
+ * @vq: the struct virtqueue
|
|
+ *
|
|
+ * After one or more virtqueue_add_buf calls, invoke this to kick
|
|
+ * the other side.
|
|
+ *
|
|
+ * Caller must ensure we don't call this with other virtqueue
|
|
+ * operations at the same time (except where noted).
|
|
+ */
|
|
+void virtqueue_kick(struct virtqueue *vq)
|
|
+{
|
|
+ if (virtqueue_kick_prepare(vq))
|
|
+ virtqueue_notify(vq);
|
|
}
|
|
EXPORT_SYMBOL_GPL(virtqueue_kick);
|
|
|
|
--- a/include/linux/virtio.h
|
|
+++ b/include/linux/virtio.h
|
|
@@ -90,6 +90,10 @@ static inline int virtqueue_add_buf(stru
|
|
|
|
void virtqueue_kick(struct virtqueue *vq);
|
|
|
|
+bool virtqueue_kick_prepare(struct virtqueue *vq);
|
|
+
|
|
+void virtqueue_notify(struct virtqueue *vq);
|
|
+
|
|
void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len);
|
|
|
|
void virtqueue_disable_cb(struct virtqueue *vq);
|