158 static void vhost_net_clear_ubuf_info(struct vhost_net *n)  in vhost_net_clear_ubuf_info()  argument
163 		kfree(n->vqs[i].ubuf_info);  in vhost_net_clear_ubuf_info()
164 		n->vqs[i].ubuf_info = NULL;  in vhost_net_clear_ubuf_info()
168 static int vhost_net_set_ubuf_info(struct vhost_net *n)  in vhost_net_set_ubuf_info()  argument
177 		n->vqs[i].ubuf_info = kmalloc(sizeof(*n->vqs[i].ubuf_info) *  in vhost_net_set_ubuf_info()
179 		if  (!n->vqs[i].ubuf_info)  in vhost_net_set_ubuf_info()
185 	vhost_net_clear_ubuf_info(n);  in vhost_net_set_ubuf_info()
189 static void vhost_net_vq_reset(struct vhost_net *n)  in vhost_net_vq_reset()  argument
193 	vhost_net_clear_ubuf_info(n);  in vhost_net_vq_reset()
196 		n->vqs[i].done_idx = 0;  in vhost_net_vq_reset()
197 		n->vqs[i].upend_idx = 0;  in vhost_net_vq_reset()
198 		n->vqs[i].ubufs = NULL;  in vhost_net_vq_reset()
199 		n->vqs[i].vhost_hlen = 0;  in vhost_net_vq_reset()
200 		n->vqs[i].sock_hlen = 0;  in vhost_net_vq_reset()
678 	struct vhost_net *n;  in vhost_net_open()  local
683 	n = kmalloc(sizeof *n, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);  in vhost_net_open()
684 	if (!n) {  in vhost_net_open()
685 		n = vmalloc(sizeof *n);  in vhost_net_open()
686 		if (!n)  in vhost_net_open()
691 		kvfree(n);  in vhost_net_open()
695 	dev = &n->dev;  in vhost_net_open()
696 	vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq;  in vhost_net_open()
697 	vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq;  in vhost_net_open()
698 	n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick;  in vhost_net_open()
699 	n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick;  in vhost_net_open()
701 		n->vqs[i].ubufs = NULL;  in vhost_net_open()
702 		n->vqs[i].ubuf_info = NULL;  in vhost_net_open()
703 		n->vqs[i].upend_idx = 0;  in vhost_net_open()
704 		n->vqs[i].done_idx = 0;  in vhost_net_open()
705 		n->vqs[i].vhost_hlen = 0;  in vhost_net_open()
706 		n->vqs[i].sock_hlen = 0;  in vhost_net_open()
710 	vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, POLLOUT, dev);  in vhost_net_open()
711 	vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, POLLIN, dev);  in vhost_net_open()
713 	f->private_data = n;  in vhost_net_open()
718 static void vhost_net_disable_vq(struct vhost_net *n,  in vhost_net_disable_vq()  argument
723 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_disable_vq()
729 static int vhost_net_enable_vq(struct vhost_net *n,  in vhost_net_enable_vq()  argument
734 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_enable_vq()
744 static struct socket *vhost_net_stop_vq(struct vhost_net *n,  in vhost_net_stop_vq()  argument
751 	vhost_net_disable_vq(n, vq);  in vhost_net_stop_vq()
757 static void vhost_net_stop(struct vhost_net *n, struct socket **tx_sock,  in vhost_net_stop()  argument
760 	*tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq);  in vhost_net_stop()
761 	*rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq);  in vhost_net_stop()
764 static void vhost_net_flush_vq(struct vhost_net *n, int index)  in vhost_net_flush_vq()  argument
766 	vhost_poll_flush(n->poll + index);  in vhost_net_flush_vq()
767 	vhost_poll_flush(&n->vqs[index].vq.poll);  in vhost_net_flush_vq()
770 static void vhost_net_flush(struct vhost_net *n)  in vhost_net_flush()  argument
772 	vhost_net_flush_vq(n, VHOST_NET_VQ_TX);  in vhost_net_flush()
773 	vhost_net_flush_vq(n, VHOST_NET_VQ_RX);  in vhost_net_flush()
774 	if (n->vqs[VHOST_NET_VQ_TX].ubufs) {  in vhost_net_flush()
775 		mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);  in vhost_net_flush()
776 		n->tx_flush = true;  in vhost_net_flush()
777 		mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);  in vhost_net_flush()
779 		vhost_net_ubuf_put_and_wait(n->vqs[VHOST_NET_VQ_TX].ubufs);  in vhost_net_flush()
780 		mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);  in vhost_net_flush()
781 		n->tx_flush = false;  in vhost_net_flush()
782 		atomic_set(&n->vqs[VHOST_NET_VQ_TX].ubufs->refcount, 1);  in vhost_net_flush()
783 		mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex);  in vhost_net_flush()
789 	struct vhost_net *n = f->private_data;  in vhost_net_release()  local
793 	vhost_net_stop(n, &tx_sock, &rx_sock);  in vhost_net_release()
794 	vhost_net_flush(n);  in vhost_net_release()
795 	vhost_dev_stop(&n->dev);  in vhost_net_release()
796 	vhost_dev_cleanup(&n->dev, false);  in vhost_net_release()
797 	vhost_net_vq_reset(n);  in vhost_net_release()
806 	vhost_net_flush(n);  in vhost_net_release()
807 	kfree(n->dev.vqs);  in vhost_net_release()
808 	kvfree(n);  in vhost_net_release()
877 static long vhost_net_set_backend(struct vhost_net *n, unsigned index, int fd)  in vhost_net_set_backend()  argument
885 	mutex_lock(&n->dev.mutex);  in vhost_net_set_backend()
886 	r = vhost_dev_check_owner(&n->dev);  in vhost_net_set_backend()
894 	vq = &n->vqs[index].vq;  in vhost_net_set_backend()
895 	nvq = &n->vqs[index];  in vhost_net_set_backend()
919 		vhost_net_disable_vq(n, vq);  in vhost_net_set_backend()
924 		r = vhost_net_enable_vq(n, vq);  in vhost_net_set_backend()
931 		n->tx_packets = 0;  in vhost_net_set_backend()
932 		n->tx_zcopy_err = 0;  in vhost_net_set_backend()
933 		n->tx_flush = false;  in vhost_net_set_backend()
941 		vhost_zerocopy_signal_used(n, vq);  in vhost_net_set_backend()
946 		vhost_net_flush_vq(n, index);  in vhost_net_set_backend()
950 	mutex_unlock(&n->dev.mutex);  in vhost_net_set_backend()
955 	vhost_net_enable_vq(n, vq);  in vhost_net_set_backend()
963 	mutex_unlock(&n->dev.mutex);  in vhost_net_set_backend()
967 static long vhost_net_reset_owner(struct vhost_net *n)  in vhost_net_reset_owner()  argument
974 	mutex_lock(&n->dev.mutex);  in vhost_net_reset_owner()
975 	err = vhost_dev_check_owner(&n->dev);  in vhost_net_reset_owner()
983 	vhost_net_stop(n, &tx_sock, &rx_sock);  in vhost_net_reset_owner()
984 	vhost_net_flush(n);  in vhost_net_reset_owner()
985 	vhost_dev_reset_owner(&n->dev, memory);  in vhost_net_reset_owner()
986 	vhost_net_vq_reset(n);  in vhost_net_reset_owner()
988 	mutex_unlock(&n->dev.mutex);  in vhost_net_reset_owner()
996 static int vhost_net_set_features(struct vhost_net *n, u64 features)  in vhost_net_set_features()  argument
1014 	mutex_lock(&n->dev.mutex);  in vhost_net_set_features()
1016 	    !vhost_log_access_ok(&n->dev)) {  in vhost_net_set_features()
1017 		mutex_unlock(&n->dev.mutex);  in vhost_net_set_features()
1021 		mutex_lock(&n->vqs[i].vq.mutex);  in vhost_net_set_features()
1022 		n->vqs[i].vq.acked_features = features;  in vhost_net_set_features()
1023 		n->vqs[i].vhost_hlen = vhost_hlen;  in vhost_net_set_features()
1024 		n->vqs[i].sock_hlen = sock_hlen;  in vhost_net_set_features()
1025 		mutex_unlock(&n->vqs[i].vq.mutex);  in vhost_net_set_features()
1027 	mutex_unlock(&n->dev.mutex);  in vhost_net_set_features()
1031 static long vhost_net_set_owner(struct vhost_net *n)  in vhost_net_set_owner()  argument
1035 	mutex_lock(&n->dev.mutex);  in vhost_net_set_owner()
1036 	if (vhost_dev_has_owner(&n->dev)) {  in vhost_net_set_owner()
1040 	r = vhost_net_set_ubuf_info(n);  in vhost_net_set_owner()
1043 	r = vhost_dev_set_owner(&n->dev);  in vhost_net_set_owner()
1045 		vhost_net_clear_ubuf_info(n);  in vhost_net_set_owner()
1046 	vhost_net_flush(n);  in vhost_net_set_owner()
1048 	mutex_unlock(&n->dev.mutex);  in vhost_net_set_owner()
1055 	struct vhost_net *n = f->private_data;  in vhost_net_ioctl()  local
1066 		return vhost_net_set_backend(n, backend.index, backend.fd);  in vhost_net_ioctl()
1077 		return vhost_net_set_features(n, features);  in vhost_net_ioctl()
1079 		return vhost_net_reset_owner(n);  in vhost_net_ioctl()
1081 		return vhost_net_set_owner(n);  in vhost_net_ioctl()
1083 		mutex_lock(&n->dev.mutex);  in vhost_net_ioctl()
1084 		r = vhost_dev_ioctl(&n->dev, ioctl, argp);  in vhost_net_ioctl()
1086 			r = vhost_vring_ioctl(&n->dev, ioctl, argp);  in vhost_net_ioctl()
1088 			vhost_net_flush(n);  in vhost_net_ioctl()
1089 		mutex_unlock(&n->dev.mutex);  in vhost_net_ioctl()