drivers/virtio/virtio_ring.c

189 	 * for split ring, it just contains last used index
200 		/* Available for split ring */
201 		struct vring_virtqueue_split split;
463 	struct vring_desc_extra *extra = vq->split.desc_extra;
524 	struct vring_desc_extra *extra = vring->split.desc_extra;
580 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
591 		desc = vq->split.vring.desc;
647 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
662 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,
674 		vq->free_head = vq->split.desc_extra[head].next;
679 	vq->split.desc_state[head].data = data;
681 		vq->split.desc_state[head].indir_desc = desc;
683 		vq->split.desc_state[head].indir_desc = ctx;
687 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
688 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
693 	vq->split.avail_idx_shadow++;
694 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
695 						vq->split.avail_idx_shadow);
745 	old = vq->split.avail_idx_shadow - vq->num_added;
746 	new = vq->split.avail_idx_shadow;
754 					vring_avail_event(&vq->split.vring)),
757 		needs_kick = !(vq->split.vring.used->flags &
772 	vq->split.desc_state[head].data = NULL;
777 	while (vq->split.vring.desc[i].flags & nextflag) {
779 		i = vq->split.desc_extra[i].next;
784 	vq->split.desc_extra[i].next = vq->free_head;
792 				vq->split.desc_state[head].indir_desc;
799 		len = vq->split.desc_extra[head].len;
801 		BUG_ON(!(vq->split.desc_extra[head].flags &
811 		vq->split.desc_state[head].indir_desc = NULL;
813 		*ctx = vq->split.desc_state[head].indir_desc;
820 			vq->split.vring.used->idx);
848 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
850 			vq->split.vring.used->ring[last_used].id);
852 			vq->split.vring.used->ring[last_used].len);
854 	if (unlikely(i >= vq->split.vring.num)) {
858 	if (unlikely(!vq->split.desc_state[i].data)) {
864 	ret = vq->split.desc_state[i].data;
870 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
872 				&vring_used_event(&vq->split.vring),
885 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
886 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
897 			vring_used_event(&vq->split.vring) = 0x0;
899 			vq->split.vring.avail->flags =
901 						vq->split.avail_flags_shadow);
917 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
918 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
920 			vq->split.vring.avail->flags =
922 						vq->split.avail_flags_shadow);
924 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
935 			vq->split.vring.used->idx);
950 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
951 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
953 			vq->split.vring.avail->flags =
955 						vq->split.avail_flags_shadow);
958 	bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
961 			&vring_used_event(&vq->split.vring),
964 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
982 	for (i = 0; i < vq->split.vring.num; i++) {
983 		if (!vq->split.desc_state[i].data)
986 		buf = vq->split.desc_state[i].data;
988 		vq->split.avail_idx_shadow--;
989 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
990 				vq->split.avail_idx_shadow);
995 	BUG_ON(vq->vq.num_free != vq->split.vring.num);
1024 	num = vq->split.vring.num;
1026 	vq->split.vring.avail->flags = 0;
1027 	vq->split.vring.avail->idx = 0;
1030 	vq->split.vring.avail->ring[num] = 0;
1032 	vq->split.vring.used->flags = 0;
1033 	vq->split.vring.used->idx = 0;
1036 	*(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0;
1040 	virtqueue_vring_init_split(&vq->split, vq);
1046 	vq->split = *vring_split;
1183 				      vq->split.vring_align,
1184 				      vq->split.may_reduce_num,
2335  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
2355  * virtqueue_notify - second half of split virtqueue_kick call.
2601 /* Only available for split ring */
2750 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
2794 	num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2848 /* Only available for split ring */
2900 					 vq->split.queue_size_in_bytes,
2901 					 vq->split.vring.desc,
2902 					 vq->split.queue_dma_addr,
2907 		kfree(vq->split.desc_state);
2908 		kfree(vq->split.desc_extra);
2937 		next = vq->split.avail_idx_shadow;
2984 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
3070 	return vq->split.queue_dma_addr;
3083 	return vq->split.queue_dma_addr +
3084 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
3097 	return vq->split.queue_dma_addr +
3098 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
3102 /* Only available for split ring */
3105 	return &to_vvq(vq)->split.vring;