104 			struct urb *urb, int is_out,
214 	struct urb		*urb = next_urb(qh);  in musb_start_urb()  local
215 	void			*buf = urb->transfer_buffer;  in musb_start_urb()
218 	unsigned		pipe = urb->pipe;  in musb_start_urb()
232 		buf = urb->setup_packet;  in musb_start_urb()
238 		offset = urb->iso_frame_desc[0].offset;  in musb_start_urb()
239 		len = urb->iso_frame_desc[0].length;  in musb_start_urb()
243 		buf = urb->transfer_buffer + urb->actual_length;  in musb_start_urb()
244 		len = urb->transfer_buffer_length - urb->actual_length;  in musb_start_urb()
248 			qh, urb, address, qh->epnum,  in musb_start_urb()
260 	musb_ep_program(musb, epnum, urb, !is_in, buf, offset, len);  in musb_start_urb()
282 			qh->frame = urb->start_frame;  in musb_start_urb()
303 static void musb_giveback(struct musb *musb, struct urb *urb, int status)  in musb_giveback()  argument
309 			urb, urb->complete, status,  in musb_giveback()
310 			usb_pipedevice(urb->pipe),  in musb_giveback()
311 			usb_pipeendpoint(urb->pipe),  in musb_giveback()
312 			usb_pipein(urb->pipe) ? "in" : "out",  in musb_giveback()
313 			urb->actual_length, urb->transfer_buffer_length  in musb_giveback()
316 	usb_hcd_unlink_urb_from_ep(musb->hcd, urb);  in musb_giveback()
318 	usb_hcd_giveback_urb(musb->hcd, urb, status);  in musb_giveback()
324 				    struct urb *urb)  in musb_save_toggle()  argument
339 	usb_settoggle(urb->dev, qh->epnum, !is_in, csr ? 1 : 0);  in musb_save_toggle()
349 static void musb_advance_schedule(struct musb *musb, struct urb *urb,  in musb_advance_schedule()  argument
357 	status = (urb->status == -EINPROGRESS) ? 0 : urb->status;  in musb_advance_schedule()
363 		musb_save_toggle(qh, is_in, urb);  in musb_advance_schedule()
366 		if (status == 0 && urb->error_count)  in musb_advance_schedule()
372 	musb_giveback(musb, urb, status);  in musb_advance_schedule()
457 musb_host_packet_rx(struct musb *musb, struct urb *urb, u8 epnum, u8 iso_err)  in musb_host_packet_rx()  argument
468 	int			pipe = urb->pipe;  in musb_host_packet_rx()
469 	void			*buffer = urb->transfer_buffer;  in musb_host_packet_rx()
474 			urb->transfer_buffer, qh->offset,  in musb_host_packet_rx()
475 			urb->transfer_buffer_length);  in musb_host_packet_rx()
484 			urb->error_count++;  in musb_host_packet_rx()
487 		d = urb->iso_frame_desc + qh->iso_idx;  in musb_host_packet_rx()
493 				urb->error_count++;  in musb_host_packet_rx()
499 		urb->actual_length += length;  in musb_host_packet_rx()
505 		done = (++qh->iso_idx >= urb->number_of_packets);  in musb_host_packet_rx()
509 		length = urb->transfer_buffer_length - qh->offset;  in musb_host_packet_rx()
511 			if (urb->status == -EINPROGRESS)  in musb_host_packet_rx()
512 				urb->status = -EOVERFLOW;  in musb_host_packet_rx()
517 		urb->actual_length += length;  in musb_host_packet_rx()
521 		done = (urb->actual_length == urb->transfer_buffer_length)  in musb_host_packet_rx()
523 			|| (urb->status != -EINPROGRESS);  in musb_host_packet_rx()
525 				&& (urb->status == -EINPROGRESS)  in musb_host_packet_rx()
526 				&& (urb->transfer_flags & URB_SHORT_NOT_OK)  in musb_host_packet_rx()
527 				&& (urb->actual_length  in musb_host_packet_rx()
528 					< urb->transfer_buffer_length))  in musb_host_packet_rx()
529 			urb->status = -EREMOTEIO;  in musb_host_packet_rx()
622 		struct urb *urb, u32 offset, u32 length)  in musb_tx_dma_program()  argument
668 	mode = (urb->transfer_flags & URB_ZERO_PACKET) ? 1 : 0;  in musb_tx_dma_program()
680 			urb->transfer_dma + offset, length)) {  in musb_tx_dma_program()
697 			struct urb *urb, int is_out,  in musb_ep_program()  argument
714 			epnum, urb, urb->dev->speed,  in musb_ep_program()
785 				if (usb_gettoggle(urb->dev, qh->epnum, 1))  in musb_ep_program()
842 					hw_ep, qh, urb, offset, len))  in musb_ep_program()
849 				sg_miter_start(&qh->sg_miter, urb->sg, 1,  in musb_ep_program()
859 				buf = qh->sg_miter.addr + urb->sg->offset +  in musb_ep_program()
860 					urb->actual_length;  in musb_ep_program()
881 			if (usb_gettoggle(urb->dev, qh->epnum, 0))  in musb_ep_program()
918 					packet_sz, !(urb->transfer_flags &  in musb_ep_program()
920 					urb->transfer_dma + offset,  in musb_ep_program()
943 	struct urb		*urb;  in musb_bulk_nak_timeout()  local
972 		urb = next_urb(cur_qh);  in musb_bulk_nak_timeout()
976 			urb->actual_length += dma->actual_len;  in musb_bulk_nak_timeout()
979 		musb_save_toggle(cur_qh, is_in, urb);  in musb_bulk_nak_timeout()
1008 static bool musb_h_ep0_continue(struct musb *musb, u16 len, struct urb *urb)  in musb_h_ep0_continue()  argument
1019 		fifo_dest = urb->transfer_buffer + urb->actual_length;  in musb_h_ep0_continue()
1020 		fifo_count = min_t(size_t, len, urb->transfer_buffer_length -  in musb_h_ep0_continue()
1021 				   urb->actual_length);  in musb_h_ep0_continue()
1023 			urb->status = -EOVERFLOW;  in musb_h_ep0_continue()
1027 		urb->actual_length += fifo_count;  in musb_h_ep0_continue()
1032 		} else if (urb->actual_length <  in musb_h_ep0_continue()
1033 				urb->transfer_buffer_length)  in musb_h_ep0_continue()
1037 		request = (struct usb_ctrlrequest *) urb->setup_packet;  in musb_h_ep0_continue()
1055 				   urb->transfer_buffer_length -  in musb_h_ep0_continue()
1056 				   urb->actual_length);  in musb_h_ep0_continue()
1058 			fifo_dest = (u8 *) (urb->transfer_buffer  in musb_h_ep0_continue()
1059 					+ urb->actual_length);  in musb_h_ep0_continue()
1066 			urb->actual_length += fifo_count;  in musb_h_ep0_continue()
1086 	struct urb		*urb;  in musb_h_ep0_irq()  local
1097 	urb = next_urb(qh);  in musb_h_ep0_irq()
1106 		csr, qh, len, urb, musb->ep0_stage);  in musb_h_ep0_irq()
1141 		if (urb)  in musb_h_ep0_irq()
1142 			urb->status = status;  in musb_h_ep0_irq()
1161 	if (unlikely(!urb)) {  in musb_h_ep0_irq()
1172 		if (musb_h_ep0_continue(musb, len, urb)) {  in musb_h_ep0_irq()
1178 			if (usb_pipeout(urb->pipe)  in musb_h_ep0_irq()
1179 					|| !urb->transfer_buffer_length)  in musb_h_ep0_irq()
1202 		musb_advance_schedule(musb, urb, hw_ep, 1);  in musb_h_ep0_irq()
1235 	struct urb		*urb = next_urb(qh);  in musb_host_tx()  local
1245 	if (!urb) {  in musb_host_tx()
1250 	pipe = urb->pipe;  in musb_host_tx()
1398 			d = urb->iso_frame_desc + qh->iso_idx;  in musb_host_tx()
1401 			if (++qh->iso_idx >= urb->number_of_packets) {  in musb_host_tx()
1408 		} else if (dma && urb->transfer_buffer_length == qh->offset) {  in musb_host_tx()
1414 			else if (qh->offset == urb->transfer_buffer_length  in musb_host_tx()
1415 					&& !(urb->transfer_flags  in musb_host_tx()
1420 				length = urb->transfer_buffer_length - offset;  in musb_host_tx()
1429 	if (urb->status != -EINPROGRESS) {  in musb_host_tx()
1432 			status = urb->status;  in musb_host_tx()
1437 		urb->status = status;  in musb_host_tx()
1438 		urb->actual_length = qh->offset;  in musb_host_tx()
1439 		musb_advance_schedule(musb, urb, hw_ep, USB_DIR_OUT);  in musb_host_tx()
1442 		if (musb_tx_dma_program(musb->dma_controller, hw_ep, qh, urb,  in musb_host_tx()
1463 	usb_hcd_unmap_urb_for_dma(musb->hcd, urb);  in musb_host_tx()
1469 	if (!urb->transfer_buffer)  in musb_host_tx()
1480 		urb->transfer_buffer = qh->sg_miter.addr;  in musb_host_tx()
1482 		musb_write_fifo(hw_ep, length, urb->transfer_buffer);  in musb_host_tx()
1486 		musb_write_fifo(hw_ep, length, urb->transfer_buffer + offset);  in musb_host_tx()
1492 		if (offset + length >= urb->transfer_buffer_length)  in musb_host_tx()
1547 	struct urb		*urb;  in musb_host_rx()  local
1563 	urb = next_urb(qh);  in musb_host_rx()
1571 	if (unlikely(!urb)) {  in musb_host_rx()
1582 	pipe = urb->pipe;  in musb_host_rx()
1585 		epnum, rx_csr, urb->actual_length,  in musb_host_rx()
1615 			if (usb_pipebulk(urb->pipe)  in musb_host_rx()
1702 			d = urb->iso_frame_desc + qh->iso_idx;  in musb_host_rx()
1711 			if (++qh->iso_idx >= urb->number_of_packets) {  in musb_host_rx()
1721 					urb->iso_frame_desc[qh->iso_idx].offset  in musb_host_rx()
1722 					+ (u32)urb->transfer_dma;  in musb_host_rx()
1725 					urb->iso_frame_desc[qh->iso_idx].length;  in musb_host_rx()
1738 			done = (urb->actual_length + xfer_len >=  in musb_host_rx()
1739 					urb->transfer_buffer_length  in musb_host_rx()
1758 	} else if (urb->status == -EINPROGRESS) {  in musb_host_rx()
1787 					(unsigned long long) urb->transfer_dma  in musb_host_rx()
1788 					+ urb->actual_length,  in musb_host_rx()
1790 					urb->transfer_buffer_length);  in musb_host_rx()
1798 				d = urb->iso_frame_desc + qh->iso_idx;  in musb_host_rx()
1802 					urb->error_count++;  in musb_host_rx()
1807 						urb->error_count++;  in musb_host_rx()
1816 				buf = urb->transfer_dma + d->offset;  in musb_host_rx()
1819 				buf = urb->transfer_dma +  in musb_host_rx()
1820 						urb->actual_length;  in musb_host_rx()
1828 			if ((urb->transfer_flags &  in musb_host_rx()
1830 				&& (urb->transfer_buffer_length -  in musb_host_rx()
1831 						urb->actual_length)  in musb_host_rx()
1838 				length = urb->transfer_buffer_length;  in musb_host_rx()
1900 			usb_hcd_unmap_urb_for_dma(musb->hcd, urb);  in musb_host_rx()
1906 			if (!urb->transfer_buffer) {  in musb_host_rx()
1908 				sg_miter_start(&qh->sg_miter, urb->sg, 1,  in musb_host_rx()
1920 				urb->transfer_buffer = qh->sg_miter.addr;  in musb_host_rx()
1921 				received_len = urb->actual_length;  in musb_host_rx()
1923 				done = musb_host_packet_rx(musb, urb, epnum,  in musb_host_rx()
1926 				received_len = urb->actual_length -  in musb_host_rx()
1931 				done = musb_host_packet_rx(musb, urb,  in musb_host_rx()
1939 	urb->actual_length += xfer_len;  in musb_host_rx()
1945 		if (urb->status == -EINPROGRESS)  in musb_host_rx()
1946 			urb->status = status;  in musb_host_rx()
1947 		musb_advance_schedule(musb, urb, hw_ep, USB_DIR_IN);  in musb_host_rx()
1968 	struct urb		*urb = next_urb(qh);  in musb_schedule()  local
2021 			toggle = usb_gettoggle(urb->dev, qh->epnum, !is_in);  in musb_schedule()
2074 	struct urb			*urb,  in musb_urb_enqueue()  argument
2079 	struct usb_host_endpoint	*hep = urb->ep;  in musb_urb_enqueue()
2091 	ret = usb_hcd_link_urb_to_ep(hcd, urb);  in musb_urb_enqueue()
2094 		urb->hcpriv = qh;  in musb_urb_enqueue()
2117 		usb_hcd_unlink_urb_from_ep(hcd, urb);  in musb_urb_enqueue()
2123 	qh->dev = urb->dev;  in musb_urb_enqueue()
2139 			ok = (usb_pipein(urb->pipe) && musb->hb_iso_rx)  in musb_urb_enqueue()
2140 				|| (usb_pipeout(urb->pipe) && musb->hb_iso_tx);  in musb_urb_enqueue()
2151 	qh->addr_reg = (u8) usb_pipedevice(urb->pipe);  in musb_urb_enqueue()
2155 	switch (urb->dev->speed) {  in musb_urb_enqueue()
2174 		if (urb->dev->speed <= USB_SPEED_FULL) {  in musb_urb_enqueue()
2204 		struct usb_device	*parent = urb->dev->parent;  in musb_urb_enqueue()
2210 			if (urb->dev->tt) {  in musb_urb_enqueue()
2211 				qh->h_port_reg = (u8) urb->dev->ttport;  in musb_urb_enqueue()
2212 				if (urb->dev->tt->hub)  in musb_urb_enqueue()
2214 						(u8) urb->dev->tt->hub->devnum;  in musb_urb_enqueue()
2215 				if (urb->dev->tt->multi)  in musb_urb_enqueue()
2238 		urb->hcpriv = qh;  in musb_urb_enqueue()
2248 		usb_hcd_unlink_urb_from_ep(hcd, urb);  in musb_urb_enqueue()
2261 static int musb_cleanup_urb(struct urb *urb, struct musb_qh *qh)  in musb_cleanup_urb()  argument
2268 	int			is_in = usb_pipein(urb->pipe);  in musb_cleanup_urb()
2283 				urb, status);  in musb_cleanup_urb()
2284 			urb->actual_length += dma->actual_len;  in musb_cleanup_urb()
2315 		musb_advance_schedule(ep->musb, urb, ep, is_in);  in musb_cleanup_urb()
2319 static int musb_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)  in musb_urb_dequeue()  argument
2324 	int			is_in  = usb_pipein(urb->pipe);  in musb_urb_dequeue()
2327 	dev_dbg(musb->controller, "urb=%p, dev%d ep%d%s\n", urb,  in musb_urb_dequeue()
2328 			usb_pipedevice(urb->pipe),  in musb_urb_dequeue()
2329 			usb_pipeendpoint(urb->pipe),  in musb_urb_dequeue()
2333 	ret = usb_hcd_check_unlink_urb(hcd, urb, status);  in musb_urb_dequeue()
2337 	qh = urb->hcpriv;  in musb_urb_dequeue()
2354 			|| urb->urb_list.prev != &qh->hep->urb_list  in musb_urb_dequeue()
2359 		musb_giveback(musb, urb, 0);  in musb_urb_dequeue()
2371 		ret = musb_cleanup_urb(urb, qh);  in musb_urb_dequeue()
2385 	struct urb		*urb;  in musb_h_disable()  local
2398 		urb = next_urb(qh);  in musb_h_disable()
2401 		if (!urb->unlinked)  in musb_h_disable()
2402 			urb->status = -ESHUTDOWN;  in musb_h_disable()
2405 		musb_cleanup_urb(urb, qh);  in musb_h_disable()
2411 			urb = next_urb(qh);  in musb_h_disable()
2412 			urb->status = -ESHUTDOWN;  in musb_h_disable()
2413 			musb_advance_schedule(musb, urb, qh->hw_ep, is_in);  in musb_h_disable()
2511 static void musb_free_temp_buffer(struct urb *urb)  in musb_free_temp_buffer()  argument
2516 	if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))  in musb_free_temp_buffer()
2519 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;  in musb_free_temp_buffer()
2521 	temp = container_of(urb->transfer_buffer, struct musb_temp_buffer,  in musb_free_temp_buffer()
2526 		       urb->transfer_buffer_length);  in musb_free_temp_buffer()
2528 	urb->transfer_buffer = temp->old_xfer_buffer;  in musb_free_temp_buffer()
2531 	urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;  in musb_free_temp_buffer()
2534 static int musb_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)  in musb_alloc_temp_buffer()  argument
2541 	if (urb->num_sgs || urb->sg ||  in musb_alloc_temp_buffer()
2542 	    urb->transfer_buffer_length == 0 ||  in musb_alloc_temp_buffer()
2543 	    !((uintptr_t)urb->transfer_buffer & (MUSB_USB_DMA_ALIGN - 1)))  in musb_alloc_temp_buffer()
2546 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;  in musb_alloc_temp_buffer()
2549 	kmalloc_size = urb->transfer_buffer_length +  in musb_alloc_temp_buffer()
2561 	temp->old_xfer_buffer = urb->transfer_buffer;  in musb_alloc_temp_buffer()
2563 		memcpy(temp->data, urb->transfer_buffer,  in musb_alloc_temp_buffer()
2564 		       urb->transfer_buffer_length);  in musb_alloc_temp_buffer()
2565 	urb->transfer_buffer = temp->data;  in musb_alloc_temp_buffer()
2567 	urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;  in musb_alloc_temp_buffer()
2572 static int musb_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,  in musb_map_urb_for_dma()  argument
2585 		return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);  in musb_map_urb_for_dma()
2587 	ret = musb_alloc_temp_buffer(urb, mem_flags);  in musb_map_urb_for_dma()
2591 	ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);  in musb_map_urb_for_dma()
2593 		musb_free_temp_buffer(urb);  in musb_map_urb_for_dma()
2598 static void musb_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)  in musb_unmap_urb_for_dma()  argument
2602 	usb_hcd_unmap_urb_for_dma(hcd, urb);  in musb_unmap_urb_for_dma()
2608 	musb_free_temp_buffer(urb);  in musb_unmap_urb_for_dma()