412 		struct urb *urb)  in xhci_urb_to_transfer_ring()  argument
414 	return xhci_triad_to_transfer_ring(xhci, urb->dev->slot_id,  in xhci_urb_to_transfer_ring()
415 		xhci_get_endpoint_index(&urb->ep->desc), urb->stream_id);  in xhci_urb_to_transfer_ring()
600 	struct urb	*urb;  in xhci_giveback_urb_in_irq()  local
603 	urb = cur_td->urb;  in xhci_giveback_urb_in_irq()
604 	urb_priv = urb->hcpriv;  in xhci_giveback_urb_in_irq()
606 	hcd = bus_to_hcd(urb->dev->bus);  in xhci_giveback_urb_in_irq()
610 		if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {  in xhci_giveback_urb_in_irq()
617 		usb_hcd_unlink_urb_from_ep(hcd, urb);  in xhci_giveback_urb_in_irq()
620 		usb_hcd_giveback_urb(hcd, urb, status);  in xhci_giveback_urb_in_irq()
678 		ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);  in xhci_handle_cmd_stop_ep()
693 					cur_td->urb,  in xhci_handle_cmd_stop_ep()
694 					cur_td->urb->stream_id);  in xhci_handle_cmd_stop_ep()
703 					cur_td->urb->stream_id,  in xhci_handle_cmd_stop_ep()
721 				ep->stopped_td->urb->stream_id, &deq_state);  in xhci_handle_cmd_stop_ep()
1808 	struct urb *urb = NULL;  in finish_td()  local
1851 	urb = td->urb;  in finish_td()
1852 	urb_priv = urb->hcpriv;  in finish_td()
1859 	if (urb->actual_length > urb->transfer_buffer_length) {  in finish_td()
1861 			urb->transfer_buffer_length,  in finish_td()
1862 			urb->actual_length);  in finish_td()
1863 		urb->actual_length = 0;  in finish_td()
1864 		if (td->urb->transfer_flags & URB_SHORT_NOT_OK)  in finish_td()
1878 		if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {  in finish_td()
1926 		if (td->urb->transfer_flags & URB_SHORT_NOT_OK)  in process_ctrl_td()
1946 			td->urb->actual_length =  in process_ctrl_td()
1947 				td->urb->transfer_buffer_length -  in process_ctrl_td()
1950 			td->urb->actual_length = 0;  in process_ctrl_td()
1965 						(td->urb->transfer_flags  in process_ctrl_td()
1971 				td->urb->actual_length =  in process_ctrl_td()
1972 					td->urb->transfer_buffer_length;  in process_ctrl_td()
1982 			td->urb->actual_length =  in process_ctrl_td()
1983 				td->urb->transfer_buffer_length -  in process_ctrl_td()
2013 	urb_priv = td->urb->hcpriv;  in process_isoc_td()
2015 	frame = &td->urb->iso_frame_desc[idx];  in process_isoc_td()
2027 		frame->status = td->urb->transfer_flags & URB_SHORT_NOT_OK ?  in process_isoc_td()
2060 		td->urb->actual_length += frame->length;  in process_isoc_td()
2074 			td->urb->actual_length += len;  in process_isoc_td()
2091 	urb_priv = td->urb->hcpriv;  in skip_isoc_td()
2093 	frame = &td->urb->iso_frame_desc[idx];  in skip_isoc_td()
2131 			if (td->urb->transfer_flags & URB_SHORT_NOT_OK)  in process_bulk_intr_td()
2142 		if (td->urb->transfer_flags & URB_SHORT_NOT_OK)  in process_bulk_intr_td()
2154 				td->urb->ep->desc.bEndpointAddress,  in process_bulk_intr_td()
2155 				td->urb->transfer_buffer_length,  in process_bulk_intr_td()
2160 			td->urb->actual_length =  in process_bulk_intr_td()
2161 				td->urb->transfer_buffer_length -  in process_bulk_intr_td()
2163 			if (td->urb->transfer_buffer_length <  in process_bulk_intr_td()
2164 					td->urb->actual_length) {  in process_bulk_intr_td()
2168 				td->urb->actual_length = 0;  in process_bulk_intr_td()
2169 				if (td->urb->transfer_flags & URB_SHORT_NOT_OK)  in process_bulk_intr_td()
2176 				if (td->urb->transfer_flags & URB_SHORT_NOT_OK)  in process_bulk_intr_td()
2182 			td->urb->actual_length =  in process_bulk_intr_td()
2183 				td->urb->transfer_buffer_length;  in process_bulk_intr_td()
2194 		td->urb->actual_length = 0;  in process_bulk_intr_td()
2200 				td->urb->actual_length +=  in process_bulk_intr_td()
2207 			td->urb->actual_length +=  in process_bulk_intr_td()
2234 	struct urb *urb = NULL;  in handle_tx_event()  local
2452 			    !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {  in handle_tx_event()
2505 		if (usb_endpoint_xfer_control(&td->urb->ep->desc))  in handle_tx_event()
2508 		else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))  in handle_tx_event()
2530 			urb = td->urb;  in handle_tx_event()
2531 			urb_priv = urb->hcpriv;  in handle_tx_event()
2535 			usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);  in handle_tx_event()
2536 			if ((urb->actual_length != urb->transfer_buffer_length &&  in handle_tx_event()
2537 						(urb->transfer_flags &  in handle_tx_event()
2540 					 !usb_endpoint_xfer_isoc(&urb->ep->desc)))  in handle_tx_event()
2543 						urb, urb->actual_length,  in handle_tx_event()
2544 						urb->transfer_buffer_length,  in handle_tx_event()
2550 			if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)  in handle_tx_event()
2552 			usb_hcd_giveback_urb(bus_to_hcd(urb->dev->bus), urb, status);  in handle_tx_event()
2855 		struct urb *urb,  in prepare_transfer()  argument
2878 	urb_priv = urb->hcpriv;  in prepare_transfer()
2885 		ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);  in prepare_transfer()
2890 	td->urb = urb;  in prepare_transfer()
2901 static unsigned int count_sg_trbs_needed(struct xhci_hcd *xhci, struct urb *urb)  in count_sg_trbs_needed()  argument
2907 	num_sgs = urb->num_mapped_sgs;  in count_sg_trbs_needed()
2908 	temp = urb->transfer_buffer_length;  in count_sg_trbs_needed()
2911 	for_each_sg(urb->sg, sg, num_sgs, i) {  in count_sg_trbs_needed()
2934 static void check_trb_math(struct urb *urb, int num_trbs, int running_total)  in check_trb_math()  argument
2937 		dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated number of "  in check_trb_math()
2939 				urb->ep->desc.bEndpointAddress, num_trbs);  in check_trb_math()
2940 	if (running_total != urb->transfer_buffer_length)  in check_trb_math()
2941 		dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "  in check_trb_math()
2944 				urb->ep->desc.bEndpointAddress,  in check_trb_math()
2946 				urb->transfer_buffer_length,  in check_trb_math()
2947 				urb->transfer_buffer_length);  in check_trb_math()
2973 		struct urb *urb, int slot_id, unsigned int ep_index)  in xhci_queue_intr_tx()  argument
2981 	ep_interval = urb->interval;  in xhci_queue_intr_tx()
2983 	if (urb->dev->speed == USB_SPEED_LOW ||  in xhci_queue_intr_tx()
2984 			urb->dev->speed == USB_SPEED_FULL)  in xhci_queue_intr_tx()
2990 		dev_dbg_ratelimited(&urb->dev->dev,  in xhci_queue_intr_tx()
2994 		urb->interval = xhci_interval;  in xhci_queue_intr_tx()
2996 		if (urb->dev->speed == USB_SPEED_LOW ||  in xhci_queue_intr_tx()
2997 				urb->dev->speed == USB_SPEED_FULL)  in xhci_queue_intr_tx()
2998 			urb->interval /= 8;  in xhci_queue_intr_tx()
3000 	return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);  in xhci_queue_intr_tx()
3025 			      struct urb *urb, unsigned int num_trbs_left)  in xhci_td_remainder()  argument
3032 	maxp = GET_MAX_PACKET(usb_endpoint_maxp(&urb->ep->desc));  in xhci_td_remainder()
3046 		struct urb *urb, int slot_id, unsigned int ep_index)  in queue_bulk_sg_tx()  argument
3065 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in queue_bulk_sg_tx()
3069 	num_trbs = count_sg_trbs_needed(xhci, urb);  in queue_bulk_sg_tx()
3070 	num_sgs = urb->num_mapped_sgs;  in queue_bulk_sg_tx()
3071 	total_packet_count = DIV_ROUND_UP(urb->transfer_buffer_length,  in queue_bulk_sg_tx()
3072 			usb_endpoint_maxp(&urb->ep->desc));  in queue_bulk_sg_tx()
3075 			ep_index, urb->stream_id,  in queue_bulk_sg_tx()
3076 			num_trbs, urb, 0, mem_flags);  in queue_bulk_sg_tx()
3080 	urb_priv = urb->hcpriv;  in queue_bulk_sg_tx()
3083 	zero_length_needed = urb->transfer_flags & URB_ZERO_PACKET &&  in queue_bulk_sg_tx()
3089 				ep_index, urb->stream_id,  in queue_bulk_sg_tx()
3090 				1, urb, 1, mem_flags);  in queue_bulk_sg_tx()
3115 	sg = urb->sg;  in queue_bulk_sg_tx()
3120 	if (trb_buff_len > urb->transfer_buffer_length)  in queue_bulk_sg_tx()
3121 		trb_buff_len = urb->transfer_buffer_length;  in queue_bulk_sg_tx()
3154 		if (usb_urb_dir_in(urb))  in queue_bulk_sg_tx()
3167 					   urb->transfer_buffer_length,  in queue_bulk_sg_tx()
3168 					   urb, num_trbs - 1);  in queue_bulk_sg_tx()
3204 		if (running_total + trb_buff_len > urb->transfer_buffer_length)  in queue_bulk_sg_tx()
3206 				urb->transfer_buffer_length - running_total;  in queue_bulk_sg_tx()
3209 	check_trb_math(urb, num_trbs, running_total);  in queue_bulk_sg_tx()
3210 	giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,  in queue_bulk_sg_tx()
3217 		struct urb *urb, int slot_id, unsigned int ep_index)  in xhci_queue_bulk_tx()  argument
3235 	if (urb->num_sgs)  in xhci_queue_bulk_tx()
3236 		return queue_bulk_sg_tx(xhci, mem_flags, urb, slot_id, ep_index);  in xhci_queue_bulk_tx()
3238 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in xhci_queue_bulk_tx()
3245 		(urb->transfer_dma & (TRB_MAX_BUFF_SIZE - 1));  in xhci_queue_bulk_tx()
3251 	if (running_total != 0 || urb->transfer_buffer_length == 0)  in xhci_queue_bulk_tx()
3254 	while (running_total < urb->transfer_buffer_length) {  in xhci_queue_bulk_tx()
3260 			ep_index, urb->stream_id,  in xhci_queue_bulk_tx()
3261 			num_trbs, urb, 0, mem_flags);  in xhci_queue_bulk_tx()
3265 	urb_priv = urb->hcpriv;  in xhci_queue_bulk_tx()
3268 	zero_length_needed = urb->transfer_flags & URB_ZERO_PACKET &&  in xhci_queue_bulk_tx()
3274 				ep_index, urb->stream_id,  in xhci_queue_bulk_tx()
3275 				1, urb, 1, mem_flags);  in xhci_queue_bulk_tx()
3291 	total_packet_count = DIV_ROUND_UP(urb->transfer_buffer_length,  in xhci_queue_bulk_tx()
3292 			usb_endpoint_maxp(&urb->ep->desc));  in xhci_queue_bulk_tx()
3294 	addr = (u64) urb->transfer_dma;  in xhci_queue_bulk_tx()
3296 		(urb->transfer_dma & (TRB_MAX_BUFF_SIZE - 1));  in xhci_queue_bulk_tx()
3297 	if (trb_buff_len > urb->transfer_buffer_length)  in xhci_queue_bulk_tx()
3298 		trb_buff_len = urb->transfer_buffer_length;  in xhci_queue_bulk_tx()
3330 		if (usb_urb_dir_in(urb))  in xhci_queue_bulk_tx()
3335 					   urb->transfer_buffer_length,  in xhci_queue_bulk_tx()
3336 					   urb, num_trbs - 1);  in xhci_queue_bulk_tx()
3356 		trb_buff_len = urb->transfer_buffer_length - running_total;  in xhci_queue_bulk_tx()
3361 	check_trb_math(urb, num_trbs, running_total);  in xhci_queue_bulk_tx()
3362 	giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,  in xhci_queue_bulk_tx()
3369 		struct urb *urb, int slot_id, unsigned int ep_index)  in xhci_queue_ctrl_tx()  argument
3381 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in xhci_queue_ctrl_tx()
3389 	if (!urb->setup_packet)  in xhci_queue_ctrl_tx()
3399 	if (urb->transfer_buffer_length > 0)  in xhci_queue_ctrl_tx()
3402 			ep_index, urb->stream_id,  in xhci_queue_ctrl_tx()
3403 			num_trbs, urb, 0, mem_flags);  in xhci_queue_ctrl_tx()
3407 	urb_priv = urb->hcpriv;  in xhci_queue_ctrl_tx()
3420 	setup = (struct usb_ctrlrequest *) urb->setup_packet;  in xhci_queue_ctrl_tx()
3428 		if (urb->transfer_buffer_length > 0) {  in xhci_queue_ctrl_tx()
3445 	if (usb_urb_dir_in(urb))  in xhci_queue_ctrl_tx()
3451 				   urb->transfer_buffer_length,  in xhci_queue_ctrl_tx()
3452 				   urb->transfer_buffer_length,  in xhci_queue_ctrl_tx()
3453 				   urb, 1);  in xhci_queue_ctrl_tx()
3455 	length_field = TRB_LEN(urb->transfer_buffer_length) |  in xhci_queue_ctrl_tx()
3459 	if (urb->transfer_buffer_length > 0) {  in xhci_queue_ctrl_tx()
3463 				lower_32_bits(urb->transfer_dma),  in xhci_queue_ctrl_tx()
3464 				upper_32_bits(urb->transfer_dma),  in xhci_queue_ctrl_tx()
3474 	if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN)  in xhci_queue_ctrl_tx()
3491 		struct urb *urb, int i)  in count_isoc_trbs_needed()  argument
3496 	addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);  in count_isoc_trbs_needed()
3497 	td_len = urb->iso_frame_desc[i].length;  in count_isoc_trbs_needed()
3517 		struct urb *urb, unsigned int total_packet_count)  in xhci_get_burst_count()  argument
3524 	max_burst = urb->ep->ss_ep_comp.bMaxBurst;  in xhci_get_burst_count()
3538 		struct urb *urb, unsigned int total_packet_count)  in xhci_get_last_burst_packet_count()  argument
3549 		max_burst = urb->ep->ss_ep_comp.bMaxBurst;  in xhci_get_last_burst_packet_count()
3566 		struct urb *urb, int slot_id, unsigned int ep_index)  in xhci_queue_isoc_tx()  argument
3583 	num_tds = urb->number_of_packets;  in xhci_queue_isoc_tx()
3589 	start_addr = (u64) urb->transfer_dma;  in xhci_queue_isoc_tx()
3593 	urb_priv = urb->hcpriv;  in xhci_queue_isoc_tx()
3602 		addr = start_addr + urb->iso_frame_desc[i].offset;  in xhci_queue_isoc_tx()
3603 		td_len = urb->iso_frame_desc[i].length;  in xhci_queue_isoc_tx()
3607 					usb_endpoint_maxp(&urb->ep->desc)));  in xhci_queue_isoc_tx()
3611 		burst_count = xhci_get_burst_count(xhci, urb->dev, urb,  in xhci_queue_isoc_tx()
3614 				urb->dev, urb, total_packet_count);  in xhci_queue_isoc_tx()
3616 		trbs_per_td = count_isoc_trbs_needed(xhci, urb, i);  in xhci_queue_isoc_tx()
3619 				urb->stream_id, trbs_per_td, urb, i, mem_flags);  in xhci_queue_isoc_tx()
3651 			if (usb_urb_dir_in(urb))  in xhci_queue_isoc_tx()
3683 						   urb, trbs_per_td - j - 1);  in xhci_queue_isoc_tx()
3714 	giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,  in xhci_queue_isoc_tx()
3737 	usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);  in xhci_queue_isoc_tx()
3749 		struct urb *urb, int slot_id, unsigned int ep_index)  in xhci_queue_isoc_tx_prepare()  argument
3765 	num_tds = urb->number_of_packets;  in xhci_queue_isoc_tx_prepare()
3767 		num_trbs += count_isoc_trbs_needed(xhci, urb, i);  in xhci_queue_isoc_tx_prepare()
3780 	urb->start_frame = start_frame;  in xhci_queue_isoc_tx_prepare()
3781 	if (urb->dev->speed == USB_SPEED_LOW ||  in xhci_queue_isoc_tx_prepare()
3782 			urb->dev->speed == USB_SPEED_FULL)  in xhci_queue_isoc_tx_prepare()
3783 		urb->start_frame >>= 3;  in xhci_queue_isoc_tx_prepare()
3786 	ep_interval = urb->interval;  in xhci_queue_isoc_tx_prepare()
3788 	if (urb->dev->speed == USB_SPEED_LOW ||  in xhci_queue_isoc_tx_prepare()
3789 			urb->dev->speed == USB_SPEED_FULL)  in xhci_queue_isoc_tx_prepare()
3795 		dev_dbg_ratelimited(&urb->dev->dev,  in xhci_queue_isoc_tx_prepare()
3799 		urb->interval = xhci_interval;  in xhci_queue_isoc_tx_prepare()
3801 		if (urb->dev->speed == USB_SPEED_LOW ||  in xhci_queue_isoc_tx_prepare()
3802 				urb->dev->speed == USB_SPEED_FULL)  in xhci_queue_isoc_tx_prepare()
3803 			urb->interval /= 8;  in xhci_queue_isoc_tx_prepare()
3807 	return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);  in xhci_queue_isoc_tx_prepare()