454 				struct spi_transfer *xfer)  in atmel_spi_use_dma()  argument
456 	return as->use_dma && xfer->len >= DMA_MIN_BYTES;  in atmel_spi_use_dma()
600 				       struct spi_transfer *xfer)  in atmel_spi_next_xfer_single()  argument
603 	unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;  in atmel_spi_next_xfer_single()
614 	if (xfer->tx_buf) {  in atmel_spi_next_xfer_single()
615 		if (xfer->bits_per_word > 8)  in atmel_spi_next_xfer_single()
616 			spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));  in atmel_spi_next_xfer_single()
618 			spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));  in atmel_spi_next_xfer_single()
625 		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,  in atmel_spi_next_xfer_single()
626 		xfer->bits_per_word);  in atmel_spi_next_xfer_single()
636 				     struct spi_transfer *xfer)  in atmel_spi_next_xfer_fifo()  argument
640 	u32 offset = xfer->len - as->current_remaining_bytes;  in atmel_spi_next_xfer_fifo()
641 	const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset);  in atmel_spi_next_xfer_fifo()
642 	const u8  *bytes = (const u8  *)((u8 *)xfer->tx_buf + offset);  in atmel_spi_next_xfer_fifo()
649 	current_remaining_data = ((xfer->bits_per_word > 8) ?  in atmel_spi_next_xfer_fifo()
668 		if (xfer->tx_buf) {  in atmel_spi_next_xfer_fifo()
669 			if (xfer->bits_per_word > 8) {  in atmel_spi_next_xfer_fifo()
686 		if (xfer->tx_buf) {  in atmel_spi_next_xfer_fifo()
687 			if (xfer->bits_per_word > 8)  in atmel_spi_next_xfer_fifo()
701 		xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,  in atmel_spi_next_xfer_fifo()
702 		xfer->bits_per_word);  in atmel_spi_next_xfer_fifo()
715 				    struct spi_transfer *xfer)  in atmel_spi_next_xfer_pio()  argument
720 		atmel_spi_next_xfer_fifo(master, xfer);  in atmel_spi_next_xfer_pio()
722 		atmel_spi_next_xfer_single(master, xfer);  in atmel_spi_next_xfer_pio()
729 				struct spi_transfer *xfer,  in atmel_spi_next_xfer_dma_submit()  argument
752 	if (xfer->rx_buf) {  in atmel_spi_next_xfer_dma_submit()
753 		as->dma.sgrx.dma_address = xfer->rx_dma + xfer->len - *plen;  in atmel_spi_next_xfer_dma_submit()
762 	if (xfer->tx_buf) {  in atmel_spi_next_xfer_dma_submit()
763 		as->dma.sgtx.dma_address = xfer->tx_dma + xfer->len - *plen;  in atmel_spi_next_xfer_dma_submit()
777 				       xfer->bits_per_word))  in atmel_spi_next_xfer_dma_submit()
795 		xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma,  in atmel_spi_next_xfer_dma_submit()
796 		xfer->rx_buf, (unsigned long long)xfer->rx_dma);  in atmel_spi_next_xfer_dma_submit()
828 				struct spi_transfer *xfer,  in atmel_spi_next_xfer_data()  argument
837 	if (xfer->rx_buf)  in atmel_spi_next_xfer_data()
838 		*rx_dma = xfer->rx_dma + xfer->len - *plen;  in atmel_spi_next_xfer_data()
845 	if (xfer->tx_buf)  in atmel_spi_next_xfer_data()
846 		*tx_dma = xfer->tx_dma + xfer->len - *plen;  in atmel_spi_next_xfer_data()
861 				    struct spi_transfer *xfer)  in atmel_spi_set_xfer_speed()  argument
875 	scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz);  in atmel_spi_set_xfer_speed()
884 			xfer->speed_hz, scbr, bus_hz/255);  in atmel_spi_set_xfer_speed()
890 			xfer->speed_hz, scbr, bus_hz);  in atmel_spi_set_xfer_speed()
906 					struct spi_transfer *xfer)  in atmel_spi_pdc_next_xfer()  argument
915 	atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);  in atmel_spi_pdc_next_xfer()
928 		xfer, xfer->len, xfer->tx_buf,  in atmel_spi_pdc_next_xfer()
929 		(unsigned long long)xfer->tx_dma, xfer->rx_buf,  in atmel_spi_pdc_next_xfer()
930 		(unsigned long long)xfer->rx_dma);  in atmel_spi_pdc_next_xfer()
934 		atmel_spi_next_xfer_data(master, xfer, &tx_dma, &rx_dma, &len);  in atmel_spi_pdc_next_xfer()
947 			xfer, xfer->len, xfer->tx_buf,  in atmel_spi_pdc_next_xfer()
948 			(unsigned long long)xfer->tx_dma, xfer->rx_buf,  in atmel_spi_pdc_next_xfer()
949 			(unsigned long long)xfer->rx_dma);  in atmel_spi_pdc_next_xfer()
974 atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_dma_map_xfer()  argument
978 	xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS;  in atmel_spi_dma_map_xfer()
979 	if (xfer->tx_buf) {  in atmel_spi_dma_map_xfer()
982 		void *nonconst_tx = (void *)xfer->tx_buf;  in atmel_spi_dma_map_xfer()
984 		xfer->tx_dma = dma_map_single(dev,  in atmel_spi_dma_map_xfer()
985 				nonconst_tx, xfer->len,  in atmel_spi_dma_map_xfer()
987 		if (dma_mapping_error(dev, xfer->tx_dma))  in atmel_spi_dma_map_xfer()
990 	if (xfer->rx_buf) {  in atmel_spi_dma_map_xfer()
991 		xfer->rx_dma = dma_map_single(dev,  in atmel_spi_dma_map_xfer()
992 				xfer->rx_buf, xfer->len,  in atmel_spi_dma_map_xfer()
994 		if (dma_mapping_error(dev, xfer->rx_dma)) {  in atmel_spi_dma_map_xfer()
995 			if (xfer->tx_buf)  in atmel_spi_dma_map_xfer()
997 						xfer->tx_dma, xfer->len,  in atmel_spi_dma_map_xfer()
1006 				     struct spi_transfer *xfer)  in atmel_spi_dma_unmap_xfer()  argument
1008 	if (xfer->tx_dma != INVALID_DMA_ADDRESS)  in atmel_spi_dma_unmap_xfer()
1009 		dma_unmap_single(master->dev.parent, xfer->tx_dma,  in atmel_spi_dma_unmap_xfer()
1010 				 xfer->len, DMA_TO_DEVICE);  in atmel_spi_dma_unmap_xfer()
1011 	if (xfer->rx_dma != INVALID_DMA_ADDRESS)  in atmel_spi_dma_unmap_xfer()
1012 		dma_unmap_single(master->dev.parent, xfer->rx_dma,  in atmel_spi_dma_unmap_xfer()
1013 				 xfer->len, DMA_FROM_DEVICE);  in atmel_spi_dma_unmap_xfer()
1022 atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_pump_single_data()  argument
1026 	unsigned long	xfer_pos = xfer->len - as->current_remaining_bytes;  in atmel_spi_pump_single_data()
1028 	if (xfer->rx_buf) {  in atmel_spi_pump_single_data()
1029 		if (xfer->bits_per_word > 8) {  in atmel_spi_pump_single_data()
1030 			rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);  in atmel_spi_pump_single_data()
1033 			rxp = ((u8 *)xfer->rx_buf) + xfer_pos;  in atmel_spi_pump_single_data()
1039 	if (xfer->bits_per_word > 8) {  in atmel_spi_pump_single_data()
1050 atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_pump_fifo_data()  argument
1054 	u32 offset = xfer->len - as->current_remaining_bytes;  in atmel_spi_pump_fifo_data()
1055 	u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset);  in atmel_spi_pump_fifo_data()
1056 	u8  *bytes = (u8  *)((u8 *)xfer->rx_buf + offset);  in atmel_spi_pump_fifo_data()
1060 	num_bytes = ((xfer->bits_per_word > 8) ?  in atmel_spi_pump_fifo_data()
1070 	if (xfer->bits_per_word > 8)  in atmel_spi_pump_fifo_data()
1076 		if (xfer->rx_buf) {  in atmel_spi_pump_fifo_data()
1077 			if (xfer->bits_per_word > 8)  in atmel_spi_pump_fifo_data()
1092 atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer)  in atmel_spi_pump_pio_data()  argument
1095 		atmel_spi_pump_fifo_data(as, xfer);  in atmel_spi_pump_pio_data()
1097 		atmel_spi_pump_single_data(as, xfer);  in atmel_spi_pump_pio_data()
1111 	struct spi_transfer	*xfer;  in atmel_spi_pio_interrupt()  local
1145 			xfer = as->current_transfer;  in atmel_spi_pio_interrupt()
1146 			atmel_spi_pump_pio_data(as, xfer);  in atmel_spi_pio_interrupt()
1279 					struct spi_transfer *xfer)  in atmel_spi_one_transfer()  argument
1292 	if (!(xfer->tx_buf || xfer->rx_buf) && xfer->len) {  in atmel_spi_one_transfer()
1299 	if (bits != xfer->bits_per_word - 8) {  in atmel_spi_one_transfer()
1310 		&& (atmel_spi_use_dma(as, xfer)	|| as->use_pdc)) {  in atmel_spi_one_transfer()
1311 		if (atmel_spi_dma_map_xfer(as, xfer) < 0)  in atmel_spi_one_transfer()
1315 	atmel_spi_set_xfer_speed(as, msg->spi, xfer);  in atmel_spi_one_transfer()
1318 	as->current_transfer = xfer;  in atmel_spi_one_transfer()
1319 	as->current_remaining_bytes = xfer->len;  in atmel_spi_one_transfer()
1324 			atmel_spi_pdc_next_xfer(master, msg, xfer);  in atmel_spi_one_transfer()
1325 		} else if (atmel_spi_use_dma(as, xfer)) {  in atmel_spi_one_transfer()
1328 								xfer, &len);  in atmel_spi_one_transfer()
1332 				atmel_spi_next_xfer_pio(master, xfer);  in atmel_spi_one_transfer()
1339 			atmel_spi_next_xfer_pio(master, xfer);  in atmel_spi_one_transfer()
1382 		} else if (atmel_spi_use_dma(as, xfer)) {  in atmel_spi_one_transfer()
1387 			&& (atmel_spi_use_dma(as, xfer) || as->use_pdc))  in atmel_spi_one_transfer()
1388 			atmel_spi_dma_unmap_xfer(master, xfer);  in atmel_spi_one_transfer()
1394 		msg->actual_length += xfer->len;  in atmel_spi_one_transfer()
1398 		&& (atmel_spi_use_dma(as, xfer) || as->use_pdc))  in atmel_spi_one_transfer()
1399 		atmel_spi_dma_unmap_xfer(master, xfer);  in atmel_spi_one_transfer()
1401 	if (xfer->delay_usecs)  in atmel_spi_one_transfer()
1402 		udelay(xfer->delay_usecs);  in atmel_spi_one_transfer()
1404 	if (xfer->cs_change) {  in atmel_spi_one_transfer()
1405 		if (list_is_last(&xfer->transfer_list,  in atmel_spi_one_transfer()
1424 	struct spi_transfer *xfer;  in atmel_spi_transfer_one_message()  local
1442 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in atmel_spi_transfer_one_message()
1443 		ret = atmel_spi_one_transfer(master, msg, xfer);  in atmel_spi_transfer_one_message()
1451 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in atmel_spi_transfer_one_message()
1454 			xfer, xfer->len,  in atmel_spi_transfer_one_message()
1455 			xfer->tx_buf, &xfer->tx_dma,  in atmel_spi_transfer_one_message()
1456 			xfer->rx_buf, &xfer->rx_dma);  in atmel_spi_transfer_one_message()