221 static inline u32 tegra_slink_readl(struct tegra_slink_data *tspi,  in tegra_slink_readl()  argument
224 	return readl(tspi->base + reg);  in tegra_slink_readl()
227 static inline void tegra_slink_writel(struct tegra_slink_data *tspi,  in tegra_slink_writel()  argument
230 	writel(val, tspi->base + reg);  in tegra_slink_writel()
234 		readl(tspi->base + SLINK_MAS_DATA);  in tegra_slink_writel()
237 static void tegra_slink_clear_status(struct tegra_slink_data *tspi)  in tegra_slink_clear_status()  argument
241 	tegra_slink_readl(tspi, SLINK_STATUS);  in tegra_slink_clear_status()
245 	tegra_slink_writel(tspi, val_write, SLINK_STATUS);  in tegra_slink_clear_status()
248 static u32 tegra_slink_get_packed_size(struct tegra_slink_data *tspi,  in tegra_slink_get_packed_size()  argument
251 	switch (tspi->bytes_per_word) {  in tegra_slink_get_packed_size()
266 	struct spi_device *spi, struct tegra_slink_data *tspi,  in tegra_slink_calculate_curr_xfer_param()  argument
269 	unsigned remain_len = t->len - tspi->cur_pos;  in tegra_slink_calculate_curr_xfer_param()
276 	tspi->bytes_per_word = DIV_ROUND_UP(bits_per_word, 8);  in tegra_slink_calculate_curr_xfer_param()
279 		tspi->is_packed = 1;  in tegra_slink_calculate_curr_xfer_param()
280 		tspi->words_per_32bit = 32/bits_per_word;  in tegra_slink_calculate_curr_xfer_param()
282 		tspi->is_packed = 0;  in tegra_slink_calculate_curr_xfer_param()
283 		tspi->words_per_32bit = 1;  in tegra_slink_calculate_curr_xfer_param()
285 	tspi->packed_size = tegra_slink_get_packed_size(tspi, t);  in tegra_slink_calculate_curr_xfer_param()
287 	if (tspi->is_packed) {  in tegra_slink_calculate_curr_xfer_param()
288 		max_len = min(remain_len, tspi->max_buf_size);  in tegra_slink_calculate_curr_xfer_param()
289 		tspi->curr_dma_words = max_len/tspi->bytes_per_word;  in tegra_slink_calculate_curr_xfer_param()
292 		max_word = (remain_len - 1) / tspi->bytes_per_word + 1;  in tegra_slink_calculate_curr_xfer_param()
293 		max_word = min(max_word, tspi->max_buf_size/4);  in tegra_slink_calculate_curr_xfer_param()
294 		tspi->curr_dma_words = max_word;  in tegra_slink_calculate_curr_xfer_param()
301 	struct tegra_slink_data *tspi, struct spi_transfer *t)  in tegra_slink_fill_tx_fifo_from_client_txbuf()  argument
310 	u8 *tx_buf = (u8 *)t->tx_buf + tspi->cur_tx_pos;  in tegra_slink_fill_tx_fifo_from_client_txbuf()
312 	fifo_status = tegra_slink_readl(tspi, SLINK_STATUS2);  in tegra_slink_fill_tx_fifo_from_client_txbuf()
315 	if (tspi->is_packed) {  in tegra_slink_fill_tx_fifo_from_client_txbuf()
316 		fifo_words_left = tx_empty_count * tspi->words_per_32bit;  in tegra_slink_fill_tx_fifo_from_client_txbuf()
317 		written_words = min(fifo_words_left, tspi->curr_dma_words);  in tegra_slink_fill_tx_fifo_from_client_txbuf()
318 		nbytes = written_words * tspi->bytes_per_word;  in tegra_slink_fill_tx_fifo_from_client_txbuf()
324 			tegra_slink_writel(tspi, x, SLINK_TX_FIFO);  in tegra_slink_fill_tx_fifo_from_client_txbuf()
327 		max_n_32bit = min(tspi->curr_dma_words,  tx_empty_count);  in tegra_slink_fill_tx_fifo_from_client_txbuf()
329 		nbytes = written_words * tspi->bytes_per_word;  in tegra_slink_fill_tx_fifo_from_client_txbuf()
332 			for (i = 0; nbytes && (i < tspi->bytes_per_word);  in tegra_slink_fill_tx_fifo_from_client_txbuf()
335 			tegra_slink_writel(tspi, x, SLINK_TX_FIFO);  in tegra_slink_fill_tx_fifo_from_client_txbuf()
338 	tspi->cur_tx_pos += written_words * tspi->bytes_per_word;  in tegra_slink_fill_tx_fifo_from_client_txbuf()
343 		struct tegra_slink_data *tspi, struct spi_transfer *t)  in tegra_slink_read_rx_fifo_to_client_rxbuf()  argument
350 	u8 *rx_buf = (u8 *)t->rx_buf + tspi->cur_rx_pos;  in tegra_slink_read_rx_fifo_to_client_rxbuf()
352 	fifo_status = tegra_slink_readl(tspi, SLINK_STATUS2);  in tegra_slink_read_rx_fifo_to_client_rxbuf()
354 	if (tspi->is_packed) {  in tegra_slink_read_rx_fifo_to_client_rxbuf()
355 		len = tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_read_rx_fifo_to_client_rxbuf()
357 			u32 x = tegra_slink_readl(tspi, SLINK_RX_FIFO);  in tegra_slink_read_rx_fifo_to_client_rxbuf()
361 		tspi->cur_rx_pos += tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_read_rx_fifo_to_client_rxbuf()
362 		read_words += tspi->curr_dma_words;  in tegra_slink_read_rx_fifo_to_client_rxbuf()
365 			u32 x = tegra_slink_readl(tspi, SLINK_RX_FIFO);  in tegra_slink_read_rx_fifo_to_client_rxbuf()
366 			for (i = 0; (i < tspi->bytes_per_word); i++)  in tegra_slink_read_rx_fifo_to_client_rxbuf()
369 		tspi->cur_rx_pos += rx_full_count * tspi->bytes_per_word;  in tegra_slink_read_rx_fifo_to_client_rxbuf()
376 		struct tegra_slink_data *tspi, struct spi_transfer *t)  in tegra_slink_copy_client_txbuf_to_spi_txbuf()  argument
379 	dma_sync_single_for_cpu(tspi->dev, tspi->tx_dma_phys,  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
380 				tspi->dma_buf_size, DMA_TO_DEVICE);  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
382 	if (tspi->is_packed) {  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
383 		unsigned len = tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
384 		memcpy(tspi->tx_dma_buf, t->tx_buf + tspi->cur_pos, len);  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
388 		u8 *tx_buf = (u8 *)t->tx_buf + tspi->cur_tx_pos;  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
389 		unsigned consume = tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
391 		for (count = 0; count < tspi->curr_dma_words; count++) {  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
393 			for (i = 0; consume && (i < tspi->bytes_per_word);  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
396 			tspi->tx_dma_buf[count] = x;  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
399 	tspi->cur_tx_pos += tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
402 	dma_sync_single_for_device(tspi->dev, tspi->tx_dma_phys,  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
403 				tspi->dma_buf_size, DMA_TO_DEVICE);  in tegra_slink_copy_client_txbuf_to_spi_txbuf()
407 		struct tegra_slink_data *tspi, struct spi_transfer *t)  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()  argument
412 	dma_sync_single_for_cpu(tspi->dev, tspi->rx_dma_phys,  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
413 		tspi->dma_buf_size, DMA_FROM_DEVICE);  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
415 	if (tspi->is_packed) {  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
416 		len = tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
417 		memcpy(t->rx_buf + tspi->cur_rx_pos, tspi->rx_dma_buf, len);  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
421 		unsigned char *rx_buf = t->rx_buf + tspi->cur_rx_pos;  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
424 		for (count = 0; count < tspi->curr_dma_words; count++) {  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
425 			u32 x = tspi->rx_dma_buf[count] & rx_mask;  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
426 			for (i = 0; (i < tspi->bytes_per_word); i++)  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
430 	tspi->cur_rx_pos += tspi->curr_dma_words * tspi->bytes_per_word;  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
433 	dma_sync_single_for_device(tspi->dev, tspi->rx_dma_phys,  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
434 		tspi->dma_buf_size, DMA_FROM_DEVICE);  in tegra_slink_copy_spi_rxbuf_to_client_rxbuf()
444 static int tegra_slink_start_tx_dma(struct tegra_slink_data *tspi, int len)  in tegra_slink_start_tx_dma()  argument
446 	reinit_completion(&tspi->tx_dma_complete);  in tegra_slink_start_tx_dma()
447 	tspi->tx_dma_desc = dmaengine_prep_slave_single(tspi->tx_dma_chan,  in tegra_slink_start_tx_dma()
448 				tspi->tx_dma_phys, len, DMA_MEM_TO_DEV,  in tegra_slink_start_tx_dma()
450 	if (!tspi->tx_dma_desc) {  in tegra_slink_start_tx_dma()
451 		dev_err(tspi->dev, "Not able to get desc for Tx\n");  in tegra_slink_start_tx_dma()
455 	tspi->tx_dma_desc->callback = tegra_slink_dma_complete;  in tegra_slink_start_tx_dma()
456 	tspi->tx_dma_desc->callback_param = &tspi->tx_dma_complete;  in tegra_slink_start_tx_dma()
458 	dmaengine_submit(tspi->tx_dma_desc);  in tegra_slink_start_tx_dma()
459 	dma_async_issue_pending(tspi->tx_dma_chan);  in tegra_slink_start_tx_dma()
463 static int tegra_slink_start_rx_dma(struct tegra_slink_data *tspi, int len)  in tegra_slink_start_rx_dma()  argument
465 	reinit_completion(&tspi->rx_dma_complete);  in tegra_slink_start_rx_dma()
466 	tspi->rx_dma_desc = dmaengine_prep_slave_single(tspi->rx_dma_chan,  in tegra_slink_start_rx_dma()
467 				tspi->rx_dma_phys, len, DMA_DEV_TO_MEM,  in tegra_slink_start_rx_dma()
469 	if (!tspi->rx_dma_desc) {  in tegra_slink_start_rx_dma()
470 		dev_err(tspi->dev, "Not able to get desc for Rx\n");  in tegra_slink_start_rx_dma()
474 	tspi->rx_dma_desc->callback = tegra_slink_dma_complete;  in tegra_slink_start_rx_dma()
475 	tspi->rx_dma_desc->callback_param = &tspi->rx_dma_complete;  in tegra_slink_start_rx_dma()
477 	dmaengine_submit(tspi->rx_dma_desc);  in tegra_slink_start_rx_dma()
478 	dma_async_issue_pending(tspi->rx_dma_chan);  in tegra_slink_start_rx_dma()
483 		struct tegra_slink_data *tspi, struct spi_transfer *t)  in tegra_slink_start_dma_based_transfer()  argument
491 	status = tegra_slink_readl(tspi, SLINK_STATUS);  in tegra_slink_start_dma_based_transfer()
493 		dev_err(tspi->dev, "Rx/Tx fifo are not empty status 0x%08x\n",  in tegra_slink_start_dma_based_transfer()
498 	val = SLINK_DMA_BLOCK_SIZE(tspi->curr_dma_words - 1);  in tegra_slink_start_dma_based_transfer()
499 	val |= tspi->packed_size;  in tegra_slink_start_dma_based_transfer()
500 	if (tspi->is_packed)  in tegra_slink_start_dma_based_transfer()
501 		len = DIV_ROUND_UP(tspi->curr_dma_words * tspi->bytes_per_word,  in tegra_slink_start_dma_based_transfer()
504 		len = tspi->curr_dma_words * 4;  in tegra_slink_start_dma_based_transfer()
514 	if (tspi->cur_direction & DATA_DIR_TX)  in tegra_slink_start_dma_based_transfer()
517 	if (tspi->cur_direction & DATA_DIR_RX)  in tegra_slink_start_dma_based_transfer()
520 	tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_dma_based_transfer()
521 	tspi->dma_control_reg = val;  in tegra_slink_start_dma_based_transfer()
523 	if (tspi->cur_direction & DATA_DIR_TX) {  in tegra_slink_start_dma_based_transfer()
524 		tegra_slink_copy_client_txbuf_to_spi_txbuf(tspi, t);  in tegra_slink_start_dma_based_transfer()
526 		ret = tegra_slink_start_tx_dma(tspi, len);  in tegra_slink_start_dma_based_transfer()
528 			dev_err(tspi->dev,  in tegra_slink_start_dma_based_transfer()
534 		status = tegra_slink_readl(tspi, SLINK_STATUS);  in tegra_slink_start_dma_based_transfer()
536 			status = tegra_slink_readl(tspi, SLINK_STATUS);  in tegra_slink_start_dma_based_transfer()
539 	if (tspi->cur_direction & DATA_DIR_RX) {  in tegra_slink_start_dma_based_transfer()
541 		dma_sync_single_for_device(tspi->dev, tspi->rx_dma_phys,  in tegra_slink_start_dma_based_transfer()
542 				tspi->dma_buf_size, DMA_FROM_DEVICE);  in tegra_slink_start_dma_based_transfer()
544 		ret = tegra_slink_start_rx_dma(tspi, len);  in tegra_slink_start_dma_based_transfer()
546 			dev_err(tspi->dev,  in tegra_slink_start_dma_based_transfer()
548 			if (tspi->cur_direction & DATA_DIR_TX)  in tegra_slink_start_dma_based_transfer()
549 				dmaengine_terminate_all(tspi->tx_dma_chan);  in tegra_slink_start_dma_based_transfer()
553 	tspi->is_curr_dma_xfer = true;  in tegra_slink_start_dma_based_transfer()
554 	if (tspi->is_packed) {  in tegra_slink_start_dma_based_transfer()
556 		tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_dma_based_transfer()
560 	tspi->dma_control_reg = val;  in tegra_slink_start_dma_based_transfer()
563 	tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_dma_based_transfer()
568 		struct tegra_slink_data *tspi, struct spi_transfer *t)  in tegra_slink_start_cpu_based_transfer()  argument
573 	val = tspi->packed_size;  in tegra_slink_start_cpu_based_transfer()
574 	if (tspi->cur_direction & DATA_DIR_TX)  in tegra_slink_start_cpu_based_transfer()
577 	if (tspi->cur_direction & DATA_DIR_RX)  in tegra_slink_start_cpu_based_transfer()
580 	tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_cpu_based_transfer()
581 	tspi->dma_control_reg = val;  in tegra_slink_start_cpu_based_transfer()
583 	if (tspi->cur_direction & DATA_DIR_TX)  in tegra_slink_start_cpu_based_transfer()
584 		cur_words = tegra_slink_fill_tx_fifo_from_client_txbuf(tspi, t);  in tegra_slink_start_cpu_based_transfer()
586 		cur_words = tspi->curr_dma_words;  in tegra_slink_start_cpu_based_transfer()
588 	tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_cpu_based_transfer()
589 	tspi->dma_control_reg = val;  in tegra_slink_start_cpu_based_transfer()
591 	tspi->is_curr_dma_xfer = false;  in tegra_slink_start_cpu_based_transfer()
592 	if (tspi->is_packed) {  in tegra_slink_start_cpu_based_transfer()
594 		tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_cpu_based_transfer()
598 	tspi->dma_control_reg = val;  in tegra_slink_start_cpu_based_transfer()
600 	tegra_slink_writel(tspi, val, SLINK_DMA_CTL);  in tegra_slink_start_cpu_based_transfer()
604 static int tegra_slink_init_dma_param(struct tegra_slink_data *tspi,  in tegra_slink_init_dma_param()  argument
613 	dma_chan = dma_request_slave_channel_reason(tspi->dev,  in tegra_slink_init_dma_param()
618 			dev_err(tspi->dev,  in tegra_slink_init_dma_param()
623 	dma_buf = dma_alloc_coherent(tspi->dev, tspi->dma_buf_size,  in tegra_slink_init_dma_param()
626 		dev_err(tspi->dev, " Not able to allocate the dma buffer\n");  in tegra_slink_init_dma_param()
632 		dma_sconfig.src_addr = tspi->phys + SLINK_RX_FIFO;  in tegra_slink_init_dma_param()
636 		dma_sconfig.dst_addr = tspi->phys + SLINK_TX_FIFO;  in tegra_slink_init_dma_param()
645 		tspi->rx_dma_chan = dma_chan;  in tegra_slink_init_dma_param()
646 		tspi->rx_dma_buf = dma_buf;  in tegra_slink_init_dma_param()
647 		tspi->rx_dma_phys = dma_phys;  in tegra_slink_init_dma_param()
649 		tspi->tx_dma_chan = dma_chan;  in tegra_slink_init_dma_param()
650 		tspi->tx_dma_buf = dma_buf;  in tegra_slink_init_dma_param()
651 		tspi->tx_dma_phys = dma_phys;  in tegra_slink_init_dma_param()
656 	dma_free_coherent(tspi->dev, tspi->dma_buf_size, dma_buf, dma_phys);  in tegra_slink_init_dma_param()
661 static void tegra_slink_deinit_dma_param(struct tegra_slink_data *tspi,  in tegra_slink_deinit_dma_param()  argument
669 		dma_buf = tspi->rx_dma_buf;  in tegra_slink_deinit_dma_param()
670 		dma_chan = tspi->rx_dma_chan;  in tegra_slink_deinit_dma_param()
671 		dma_phys = tspi->rx_dma_phys;  in tegra_slink_deinit_dma_param()
672 		tspi->rx_dma_chan = NULL;  in tegra_slink_deinit_dma_param()
673 		tspi->rx_dma_buf = NULL;  in tegra_slink_deinit_dma_param()
675 		dma_buf = tspi->tx_dma_buf;  in tegra_slink_deinit_dma_param()
676 		dma_chan = tspi->tx_dma_chan;  in tegra_slink_deinit_dma_param()
677 		dma_phys = tspi->tx_dma_phys;  in tegra_slink_deinit_dma_param()
678 		tspi->tx_dma_buf = NULL;  in tegra_slink_deinit_dma_param()
679 		tspi->tx_dma_chan = NULL;  in tegra_slink_deinit_dma_param()
684 	dma_free_coherent(tspi->dev, tspi->dma_buf_size, dma_buf, dma_phys);  in tegra_slink_deinit_dma_param()
691 	struct tegra_slink_data *tspi = spi_master_get_devdata(spi->master);  in tegra_slink_start_transfer_one()  local
701 	if (speed != tspi->cur_speed) {  in tegra_slink_start_transfer_one()
702 		clk_set_rate(tspi->clk, speed * 4);  in tegra_slink_start_transfer_one()
703 		tspi->cur_speed = speed;  in tegra_slink_start_transfer_one()
706 	tspi->cur_spi = spi;  in tegra_slink_start_transfer_one()
707 	tspi->cur_pos = 0;  in tegra_slink_start_transfer_one()
708 	tspi->cur_rx_pos = 0;  in tegra_slink_start_transfer_one()
709 	tspi->cur_tx_pos = 0;  in tegra_slink_start_transfer_one()
710 	tspi->curr_xfer = t;  in tegra_slink_start_transfer_one()
711 	total_fifo_words = tegra_slink_calculate_curr_xfer_param(spi, tspi, t);  in tegra_slink_start_transfer_one()
713 	command = tspi->command_reg;  in tegra_slink_start_transfer_one()
717 	command2 = tspi->command2_reg;  in tegra_slink_start_transfer_one()
720 	tegra_slink_writel(tspi, command, SLINK_COMMAND);  in tegra_slink_start_transfer_one()
721 	tspi->command_reg = command;  in tegra_slink_start_transfer_one()
723 	tspi->cur_direction = 0;  in tegra_slink_start_transfer_one()
726 		tspi->cur_direction |= DATA_DIR_RX;  in tegra_slink_start_transfer_one()
730 		tspi->cur_direction |= DATA_DIR_TX;  in tegra_slink_start_transfer_one()
732 	tegra_slink_writel(tspi, command2, SLINK_COMMAND2);  in tegra_slink_start_transfer_one()
733 	tspi->command2_reg = command2;  in tegra_slink_start_transfer_one()
736 		ret = tegra_slink_start_dma_based_transfer(tspi, t);  in tegra_slink_start_transfer_one()
738 		ret = tegra_slink_start_cpu_based_transfer(tspi, t);  in tegra_slink_start_transfer_one()
751 	struct tegra_slink_data *tspi = spi_master_get_devdata(spi->master);  in tegra_slink_setup()  local
762 	ret = pm_runtime_get_sync(tspi->dev);  in tegra_slink_setup()
764 		dev_err(tspi->dev, "pm runtime failed, e = %d\n", ret);  in tegra_slink_setup()
768 	spin_lock_irqsave(&tspi->lock, flags);  in tegra_slink_setup()
769 	val = tspi->def_command_reg;  in tegra_slink_setup()
774 	tspi->def_command_reg = val;  in tegra_slink_setup()
775 	tegra_slink_writel(tspi, tspi->def_command_reg, SLINK_COMMAND);  in tegra_slink_setup()
776 	spin_unlock_irqrestore(&tspi->lock, flags);  in tegra_slink_setup()
778 	pm_runtime_put(tspi->dev);  in tegra_slink_setup()
785 	struct tegra_slink_data *tspi = spi_master_get_devdata(master);  in tegra_slink_prepare_message()  local
788 	tegra_slink_clear_status(tspi);  in tegra_slink_prepare_message()
790 	tspi->command_reg = tspi->def_command_reg;  in tegra_slink_prepare_message()
791 	tspi->command_reg |= SLINK_CS_SW | SLINK_CS_VALUE;  in tegra_slink_prepare_message()
793 	tspi->command2_reg = tspi->def_command2_reg;  in tegra_slink_prepare_message()
794 	tspi->command2_reg |= SLINK_SS_EN_CS(spi->chip_select);  in tegra_slink_prepare_message()
796 	tspi->command_reg &= ~SLINK_MODES;  in tegra_slink_prepare_message()
798 		tspi->command_reg |= SLINK_CK_SDA;  in tegra_slink_prepare_message()
801 		tspi->command_reg |= SLINK_IDLE_SCLK_DRIVE_HIGH;  in tegra_slink_prepare_message()
803 		tspi->command_reg |= SLINK_IDLE_SCLK_DRIVE_LOW;  in tegra_slink_prepare_message()
812 	struct tegra_slink_data *tspi = spi_master_get_devdata(master);  in tegra_slink_transfer_one()  local
815 	reinit_completion(&tspi->xfer_completion);  in tegra_slink_transfer_one()
818 		dev_err(tspi->dev,  in tegra_slink_transfer_one()
823 	ret = wait_for_completion_timeout(&tspi->xfer_completion,  in tegra_slink_transfer_one()
826 		dev_err(tspi->dev,  in tegra_slink_transfer_one()
831 	if (tspi->tx_status)  in tegra_slink_transfer_one()
832 		return tspi->tx_status;  in tegra_slink_transfer_one()
833 	if (tspi->rx_status)  in tegra_slink_transfer_one()
834 		return tspi->rx_status;  in tegra_slink_transfer_one()
842 	struct tegra_slink_data *tspi = spi_master_get_devdata(master);  in tegra_slink_unprepare_message()  local
844 	tegra_slink_writel(tspi, tspi->def_command_reg, SLINK_COMMAND);  in tegra_slink_unprepare_message()
845 	tegra_slink_writel(tspi, tspi->def_command2_reg, SLINK_COMMAND2);  in tegra_slink_unprepare_message()
850 static irqreturn_t handle_cpu_based_xfer(struct tegra_slink_data *tspi)  in handle_cpu_based_xfer()  argument
852 	struct spi_transfer *t = tspi->curr_xfer;  in handle_cpu_based_xfer()
855 	spin_lock_irqsave(&tspi->lock, flags);  in handle_cpu_based_xfer()
856 	if (tspi->tx_status ||  tspi->rx_status ||  in handle_cpu_based_xfer()
857 				(tspi->status_reg & SLINK_BSY)) {  in handle_cpu_based_xfer()
858 		dev_err(tspi->dev,  in handle_cpu_based_xfer()
859 			"CpuXfer ERROR bit set 0x%x\n", tspi->status_reg);  in handle_cpu_based_xfer()
860 		dev_err(tspi->dev,  in handle_cpu_based_xfer()
861 			"CpuXfer 0x%08x:0x%08x:0x%08x\n", tspi->command_reg,  in handle_cpu_based_xfer()
862 				tspi->command2_reg, tspi->dma_control_reg);  in handle_cpu_based_xfer()
863 		reset_control_assert(tspi->rst);  in handle_cpu_based_xfer()
865 		reset_control_deassert(tspi->rst);  in handle_cpu_based_xfer()
866 		complete(&tspi->xfer_completion);  in handle_cpu_based_xfer()
870 	if (tspi->cur_direction & DATA_DIR_RX)  in handle_cpu_based_xfer()
871 		tegra_slink_read_rx_fifo_to_client_rxbuf(tspi, t);  in handle_cpu_based_xfer()
873 	if (tspi->cur_direction & DATA_DIR_TX)  in handle_cpu_based_xfer()
874 		tspi->cur_pos = tspi->cur_tx_pos;  in handle_cpu_based_xfer()
876 		tspi->cur_pos = tspi->cur_rx_pos;  in handle_cpu_based_xfer()
878 	if (tspi->cur_pos == t->len) {  in handle_cpu_based_xfer()
879 		complete(&tspi->xfer_completion);  in handle_cpu_based_xfer()
883 	tegra_slink_calculate_curr_xfer_param(tspi->cur_spi, tspi, t);  in handle_cpu_based_xfer()
884 	tegra_slink_start_cpu_based_transfer(tspi, t);  in handle_cpu_based_xfer()
886 	spin_unlock_irqrestore(&tspi->lock, flags);  in handle_cpu_based_xfer()
890 static irqreturn_t handle_dma_based_xfer(struct tegra_slink_data *tspi)  in handle_dma_based_xfer()  argument
892 	struct spi_transfer *t = tspi->curr_xfer;  in handle_dma_based_xfer()
899 	if (tspi->cur_direction & DATA_DIR_TX) {  in handle_dma_based_xfer()
900 		if (tspi->tx_status) {  in handle_dma_based_xfer()
901 			dmaengine_terminate_all(tspi->tx_dma_chan);  in handle_dma_based_xfer()
905 				&tspi->tx_dma_complete, SLINK_DMA_TIMEOUT);  in handle_dma_based_xfer()
907 				dmaengine_terminate_all(tspi->tx_dma_chan);  in handle_dma_based_xfer()
908 				dev_err(tspi->dev, "TxDma Xfer failed\n");  in handle_dma_based_xfer()
914 	if (tspi->cur_direction & DATA_DIR_RX) {  in handle_dma_based_xfer()
915 		if (tspi->rx_status) {  in handle_dma_based_xfer()
916 			dmaengine_terminate_all(tspi->rx_dma_chan);  in handle_dma_based_xfer()
920 				&tspi->rx_dma_complete, SLINK_DMA_TIMEOUT);  in handle_dma_based_xfer()
922 				dmaengine_terminate_all(tspi->rx_dma_chan);  in handle_dma_based_xfer()
923 				dev_err(tspi->dev, "RxDma Xfer failed\n");  in handle_dma_based_xfer()
929 	spin_lock_irqsave(&tspi->lock, flags);  in handle_dma_based_xfer()
931 		dev_err(tspi->dev,  in handle_dma_based_xfer()
932 			"DmaXfer: ERROR bit set 0x%x\n", tspi->status_reg);  in handle_dma_based_xfer()
933 		dev_err(tspi->dev,  in handle_dma_based_xfer()
934 			"DmaXfer 0x%08x:0x%08x:0x%08x\n", tspi->command_reg,  in handle_dma_based_xfer()
935 				tspi->command2_reg, tspi->dma_control_reg);  in handle_dma_based_xfer()
936 		reset_control_assert(tspi->rst);  in handle_dma_based_xfer()
938 		reset_control_assert(tspi->rst);  in handle_dma_based_xfer()
939 		complete(&tspi->xfer_completion);  in handle_dma_based_xfer()
940 		spin_unlock_irqrestore(&tspi->lock, flags);  in handle_dma_based_xfer()
944 	if (tspi->cur_direction & DATA_DIR_RX)  in handle_dma_based_xfer()
945 		tegra_slink_copy_spi_rxbuf_to_client_rxbuf(tspi, t);  in handle_dma_based_xfer()
947 	if (tspi->cur_direction & DATA_DIR_TX)  in handle_dma_based_xfer()
948 		tspi->cur_pos = tspi->cur_tx_pos;  in handle_dma_based_xfer()
950 		tspi->cur_pos = tspi->cur_rx_pos;  in handle_dma_based_xfer()
952 	if (tspi->cur_pos == t->len) {  in handle_dma_based_xfer()
953 		complete(&tspi->xfer_completion);  in handle_dma_based_xfer()
958 	total_fifo_words = tegra_slink_calculate_curr_xfer_param(tspi->cur_spi,  in handle_dma_based_xfer()
959 							tspi, t);  in handle_dma_based_xfer()
961 		err = tegra_slink_start_dma_based_transfer(tspi, t);  in handle_dma_based_xfer()
963 		err = tegra_slink_start_cpu_based_transfer(tspi, t);  in handle_dma_based_xfer()
966 	spin_unlock_irqrestore(&tspi->lock, flags);  in handle_dma_based_xfer()
972 	struct tegra_slink_data *tspi = context_data;  in tegra_slink_isr_thread()  local
974 	if (!tspi->is_curr_dma_xfer)  in tegra_slink_isr_thread()
975 		return handle_cpu_based_xfer(tspi);  in tegra_slink_isr_thread()
976 	return handle_dma_based_xfer(tspi);  in tegra_slink_isr_thread()
981 	struct tegra_slink_data *tspi = context_data;  in tegra_slink_isr()  local
983 	tspi->status_reg = tegra_slink_readl(tspi, SLINK_STATUS);  in tegra_slink_isr()
984 	if (tspi->cur_direction & DATA_DIR_TX)  in tegra_slink_isr()
985 		tspi->tx_status = tspi->status_reg &  in tegra_slink_isr()
988 	if (tspi->cur_direction & DATA_DIR_RX)  in tegra_slink_isr()
989 		tspi->rx_status = tspi->status_reg &  in tegra_slink_isr()
991 	tegra_slink_clear_status(tspi);  in tegra_slink_isr()
1014 	struct tegra_slink_data	*tspi;  in tegra_slink_probe()  local
1027 	master = spi_alloc_master(&pdev->dev, sizeof(*tspi));  in tegra_slink_probe()
1043 	tspi = spi_master_get_devdata(master);  in tegra_slink_probe()
1044 	tspi->master = master;  in tegra_slink_probe()
1045 	tspi->dev = &pdev->dev;  in tegra_slink_probe()
1046 	tspi->chip_data = cdata;  in tegra_slink_probe()
1047 	spin_lock_init(&tspi->lock);  in tegra_slink_probe()
1049 	if (of_property_read_u32(tspi->dev->of_node, "spi-max-frequency",  in tegra_slink_probe()
1059 	tspi->phys = r->start;  in tegra_slink_probe()
1060 	tspi->base = devm_ioremap_resource(&pdev->dev, r);  in tegra_slink_probe()
1061 	if (IS_ERR(tspi->base)) {  in tegra_slink_probe()
1062 		ret = PTR_ERR(tspi->base);  in tegra_slink_probe()
1067 	tspi->irq = spi_irq;  in tegra_slink_probe()
1068 	ret = request_threaded_irq(tspi->irq, tegra_slink_isr,  in tegra_slink_probe()
1070 			dev_name(&pdev->dev), tspi);  in tegra_slink_probe()
1073 					tspi->irq);  in tegra_slink_probe()
1077 	tspi->clk = devm_clk_get(&pdev->dev, NULL);  in tegra_slink_probe()
1078 	if (IS_ERR(tspi->clk)) {  in tegra_slink_probe()
1080 		ret = PTR_ERR(tspi->clk);  in tegra_slink_probe()
1084 	tspi->rst = devm_reset_control_get(&pdev->dev, "spi");  in tegra_slink_probe()
1085 	if (IS_ERR(tspi->rst)) {  in tegra_slink_probe()
1087 		ret = PTR_ERR(tspi->rst);  in tegra_slink_probe()
1091 	tspi->max_buf_size = SLINK_FIFO_DEPTH << 2;  in tegra_slink_probe()
1092 	tspi->dma_buf_size = DEFAULT_SPI_DMA_BUF_LEN;  in tegra_slink_probe()
1094 	ret = tegra_slink_init_dma_param(tspi, true);  in tegra_slink_probe()
1097 	ret = tegra_slink_init_dma_param(tspi, false);  in tegra_slink_probe()
1100 	tspi->max_buf_size = tspi->dma_buf_size;  in tegra_slink_probe()
1101 	init_completion(&tspi->tx_dma_complete);  in tegra_slink_probe()
1102 	init_completion(&tspi->rx_dma_complete);  in tegra_slink_probe()
1104 	init_completion(&tspi->xfer_completion);  in tegra_slink_probe()
1118 	tspi->def_command_reg  = SLINK_M_S;  in tegra_slink_probe()
1119 	tspi->def_command2_reg = SLINK_CS_ACTIVE_BETWEEN;  in tegra_slink_probe()
1120 	tegra_slink_writel(tspi, tspi->def_command_reg, SLINK_COMMAND);  in tegra_slink_probe()
1121 	tegra_slink_writel(tspi, tspi->def_command2_reg, SLINK_COMMAND2);  in tegra_slink_probe()
1136 	tegra_slink_deinit_dma_param(tspi, false);  in tegra_slink_probe()
1138 	tegra_slink_deinit_dma_param(tspi, true);  in tegra_slink_probe()
1140 	free_irq(spi_irq, tspi);  in tegra_slink_probe()
1149 	struct tegra_slink_data	*tspi = spi_master_get_devdata(master);  in tegra_slink_remove()  local
1151 	free_irq(tspi->irq, tspi);  in tegra_slink_remove()
1153 	if (tspi->tx_dma_chan)  in tegra_slink_remove()
1154 		tegra_slink_deinit_dma_param(tspi, false);  in tegra_slink_remove()
1156 	if (tspi->rx_dma_chan)  in tegra_slink_remove()
1157 		tegra_slink_deinit_dma_param(tspi, true);  in tegra_slink_remove()
1177 	struct tegra_slink_data *tspi = spi_master_get_devdata(master);  in tegra_slink_resume()  local
1185 	tegra_slink_writel(tspi, tspi->command_reg, SLINK_COMMAND);  in tegra_slink_resume()
1186 	tegra_slink_writel(tspi, tspi->command2_reg, SLINK_COMMAND2);  in tegra_slink_resume()
1196 	struct tegra_slink_data *tspi = spi_master_get_devdata(master);  in tegra_slink_runtime_suspend()  local
1199 	tegra_slink_readl(tspi, SLINK_MAS_DATA);  in tegra_slink_runtime_suspend()
1201 	clk_disable_unprepare(tspi->clk);  in tegra_slink_runtime_suspend()
1208 	struct tegra_slink_data *tspi = spi_master_get_devdata(master);  in tegra_slink_runtime_resume()  local
1211 	ret = clk_prepare_enable(tspi->clk);  in tegra_slink_runtime_resume()
1213 		dev_err(tspi->dev, "clk_prepare failed: %d\n", ret);  in tegra_slink_runtime_resume()