Lines Matching refs:vq
47 dev_err(&(_vq)->vq.vdev->dev, \
48 "%s:"fmt, (_vq)->vq.name, ##args); \
56 (_vq)->vq.name, (_vq)->in_use); \
64 dev_err(&_vq->vq.vdev->dev, \
65 "%s:"fmt, (_vq)->vq.name, ##args); \
68 #define START_USE(vq)
69 #define END_USE(vq)
74 struct virtqueue vq;
95 void (*notify)(struct virtqueue *vq);
106 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
109 static int vring_add_indirect(struct vring_virtqueue *vq,
144 vq->num_free--;
147 head = vq->free_head;
148 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
149 vq->vring.desc[head].addr = virt_to_phys(desc);
150 vq->vring.desc[head].len = i * sizeof(struct vring_desc);
153 vq->free_head = vq->vring.desc[head].next;
165 struct vring_virtqueue *vq = to_vvq(_vq);
169 START_USE(vq);
173 if (vq->indirect && (out + in) > 1 && vq->num_free) {
174 head = vring_add_indirect(vq, sg, out, in, gfp);
179 BUG_ON(out + in > vq->vring.num);
182 if (vq->num_free < out + in) {
184 out + in, vq->num_free);
186 vq->notify(&vq->vq);
187 END_USE(vq);
192 vq->num_free -= out + in;
194 head = vq->free_head;
195 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
196 vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
197 vq->vring.desc[i].addr = sg_phys(sg);
198 vq->vring.desc[i].len = sg->length;
202 for (; in; i = vq->vring.desc[i].next, in--) {
203 vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
204 vq->vring.desc[i].addr = sg_phys(sg);
205 vq->vring.desc[i].len = sg->length;
210 vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT;
213 vq->free_head = i;
217 vq->data[head] = data;
219 avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num;
220 vq->vring.avail->ring[avail] = head;
222 pr_debug("Added buffer head %i to %p\n", head, vq);
223 END_USE(vq);
226 if (vq->indirect)
227 return vq->num_free ? vq->vring.num : 0;
228 return vq->num_free;
234 struct vring_virtqueue *vq = to_vvq(_vq);
235 START_USE(vq);
240 vq->vring.avail->idx += vq->num_added;
241 vq->num_added = 0;
246 if (!(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY))
248 vq->notify(&vq->vq);
250 END_USE(vq);
254 static void detach_buf(struct vring_virtqueue *vq, unsigned int head)
259 vq->data[head] = NULL;
265 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
266 kfree(phys_to_virt(vq->vring.desc[i].addr));
268 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
269 i = vq->vring.desc[i].next;
270 vq->num_free++;
273 vq->vring.desc[i].next = vq->free_head;
274 vq->free_head = head;
276 vq->num_free++;
279 static inline bool more_used(const struct vring_virtqueue *vq)
281 return vq->last_used_idx != vq->vring.used->idx;
286 struct vring_virtqueue *vq = to_vvq(_vq);
290 START_USE(vq);
292 if (unlikely(vq->broken)) {
293 END_USE(vq);
297 if (!more_used(vq)) {
299 END_USE(vq);
306 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id;
307 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len;
309 if (unlikely(i >= vq->vring.num)) {
310 BAD_RING(vq, "id %u out of range\n", i);
313 if (unlikely(!vq->data[i])) {
314 BAD_RING(vq, "id %u is not a head!\n", i);
319 ret = vq->data[i];
320 detach_buf(vq, i);
321 vq->last_used_idx++;
322 END_USE(vq);
329 struct vring_virtqueue *vq = to_vvq(_vq);
331 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
337 struct vring_virtqueue *vq = to_vvq(_vq);
339 START_USE(vq);
343 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
345 if (unlikely(more_used(vq))) {
346 END_USE(vq);
350 END_USE(vq);
357 struct vring_virtqueue *vq = to_vvq(_vq);
361 START_USE(vq);
363 for (i = 0; i < vq->vring.num; i++) {
364 if (!vq->data[i])
367 buf = vq->data[i];
368 detach_buf(vq, i);
369 END_USE(vq);
373 BUG_ON(vq->num_free != vq->vring.num);
375 END_USE(vq);
382 struct vring_virtqueue *vq = to_vvq(_vq);
384 if (!more_used(vq)) {
385 pr_debug("virtqueue interrupt with no work for %p\n", vq);
389 if (unlikely(vq->broken))
392 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
393 if (vq->vq.callback)
394 vq->vq.callback(&vq->vq);
408 struct vring_virtqueue *vq;
417 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
418 if (!vq)
421 vring_init(&vq->vring, num, pages, vring_align);
422 vq->vq.callback = callback;
423 vq->vq.vdev = vdev;
424 vq->vq.name = name;
425 vq->notify = notify;
426 vq->broken = false;
427 vq->last_used_idx = 0;
428 vq->num_added = 0;
429 list_add_tail(&vq->vq.list, &vdev->vqs);
431 vq->in_use = false;
434 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
438 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
441 vq->num_free = num;
442 vq->free_head = 0;
444 vq->vring.desc[i].next = i+1;
445 vq->data[i] = NULL;
447 vq->data[i] = NULL;
449 return &vq->vq;
453 void vring_del_virtqueue(struct virtqueue *vq)
455 list_del(&vq->list);
456 kfree(to_vvq(vq));