61 	struct vring vring;  member
166 	BUG_ON(total_sg > vq->vring.num);  in virtqueue_add()
180 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);  in virtqueue_add()
181 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, virt_to_phys(desc));  in virtqueue_add()
184 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));  in virtqueue_add()
191 		desc = vq->vring.desc;  in virtqueue_add()
235 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);  in virtqueue_add()
244 	avail = vq->avail_idx_shadow & (vq->vring.num - 1);  in virtqueue_add()
245 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add()
251 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_add()
377 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),  in virtqueue_kick_prepare()
380 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));  in virtqueue_kick_prepare()
442 	if (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT))  in detach_buf()
443 		kfree(phys_to_virt(virtio64_to_cpu(vq->vq.vdev, vq->vring.desc[i].addr)));  in detach_buf()
445 	while (vq->vring.desc[i].flags & cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT)) {  in detach_buf()
446 		i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);  in detach_buf()
450 	vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);  in detach_buf()
458 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);  in more_used()
500 	last_used = (vq->last_used_idx & (vq->vring.num - 1));  in virtqueue_get_buf()
501 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);  in virtqueue_get_buf()
502 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);  in virtqueue_get_buf()
504 	if (unlikely(i >= vq->vring.num)) {  in virtqueue_get_buf()
521 		vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx);  in virtqueue_get_buf()
549 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_disable_cb()
581 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_prepare()
583 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);  in virtqueue_enable_cb_prepare()
603 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);  in virtqueue_poll()
652 		vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);  in virtqueue_enable_cb_delayed()
656 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs);  in virtqueue_enable_cb_delayed()
658 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {  in virtqueue_enable_cb_delayed()
684 	for (i = 0; i < vq->vring.num; i++) {  in virtqueue_detach_unused_buf()
691 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);  in virtqueue_detach_unused_buf()
696 	BUG_ON(vq->vq.num_free != vq->vring.num);  in virtqueue_detach_unused_buf()
746 	vring_init(&vq->vring, num, pages, vring_align);  in vring_new_virtqueue()
771 		vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);  in vring_new_virtqueue()
777 		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);  in vring_new_virtqueue()
826 	return vq->vring.num;  in virtqueue_get_vring_size()
857 	return vq->vring.avail;  in virtqueue_get_avail()
865 	return vq->vring.used;  in virtqueue_get_used()