133 static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)		\
135 	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);	\
137 	if (spi_imx->rx_buf) {						\
138 		*(type *)spi_imx->rx_buf = val;				\
139 		spi_imx->rx_buf += sizeof(type);			\
144 static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)		\
148 	if (spi_imx->tx_buf) {						\
149 		val = *(type *)spi_imx->tx_buf;				\
150 		spi_imx->tx_buf += sizeof(type);			\
153 	spi_imx->count -= sizeof(type);					\
155 	writel(val, spi_imx->base + MXC_CSPITXDATA);			\
202 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_can_dma()  local
204 	if (spi_imx->dma_is_inited  in spi_imx_can_dma()
205 	    && transfer->len > spi_imx->rx_wml * sizeof(u32)  in spi_imx_can_dma()
206 	    && transfer->len > spi_imx->tx_wml * sizeof(u32))  in spi_imx_can_dma()
285 static void __maybe_unused mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx51_ecspi_intctrl()  argument
295 	writel(val, spi_imx->base + MX51_ECSPI_INT);  in mx51_ecspi_intctrl()
298 static void __maybe_unused mx51_ecspi_trigger(struct spi_imx_data *spi_imx)  in mx51_ecspi_trigger()  argument
300 	u32 reg = readl(spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_trigger()
302 	if (!spi_imx->usedma)  in mx51_ecspi_trigger()
304 	else if (!spi_imx->dma_finished)  in mx51_ecspi_trigger()
308 	writel(reg, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_trigger()
311 static int __maybe_unused mx51_ecspi_config(struct spi_imx_data *spi_imx,  in mx51_ecspi_config()  argument
328 	ctrl |= mx51_ecspi_clkdiv(spi_imx->spi_clk, config->speed_hz, &clk);  in mx51_ecspi_config()
354 	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_config()
355 	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);  in mx51_ecspi_config()
378 	if (spi_imx->dma_is_inited) {  in mx51_ecspi_config()
379 		dma = readl(spi_imx->base + MX51_ECSPI_DMA);  in mx51_ecspi_config()
381 		spi_imx->rxt_wml = spi_imx_get_fifosize(spi_imx) / 2;  in mx51_ecspi_config()
382 		rx_wml_cfg = spi_imx->rx_wml << MX51_ECSPI_DMA_RX_WML_OFFSET;  in mx51_ecspi_config()
383 		tx_wml_cfg = spi_imx->tx_wml << MX51_ECSPI_DMA_TX_WML_OFFSET;  in mx51_ecspi_config()
384 		rxt_wml_cfg = spi_imx->rxt_wml << MX51_ECSPI_DMA_RXT_WML_OFFSET;  in mx51_ecspi_config()
393 		writel(dma, spi_imx->base + MX51_ECSPI_DMA);  in mx51_ecspi_config()
399 static int __maybe_unused mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)  in mx51_ecspi_rx_available()  argument
401 	return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;  in mx51_ecspi_rx_available()
404 static void __maybe_unused mx51_ecspi_reset(struct spi_imx_data *spi_imx)  in mx51_ecspi_reset()  argument
407 	while (mx51_ecspi_rx_available(spi_imx))  in mx51_ecspi_reset()
408 		readl(spi_imx->base + MXC_CSPIRXDATA);  in mx51_ecspi_reset()
434 static void __maybe_unused mx31_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx31_intctrl()  argument
443 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx31_intctrl()
446 static void __maybe_unused mx31_trigger(struct spi_imx_data *spi_imx)  in mx31_trigger()  argument
450 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx31_trigger()
452 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx31_trigger()
455 static int __maybe_unused mx31_config(struct spi_imx_data *spi_imx,  in mx31_config()  argument
459 	int cs = spi_imx->chipselect[config->cs];  in mx31_config()
461 	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz) <<  in mx31_config()
464 	if (is_imx35_cspi(spi_imx)) {  in mx31_config()
479 			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :  in mx31_config()
482 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx31_config()
487 static int __maybe_unused mx31_rx_available(struct spi_imx_data *spi_imx)  in mx31_rx_available()  argument
489 	return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;  in mx31_rx_available()
492 static void __maybe_unused mx31_reset(struct spi_imx_data *spi_imx)  in mx31_reset()  argument
495 	while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)  in mx31_reset()
496 		readl(spi_imx->base + MXC_CSPIRXDATA);  in mx31_reset()
512 static void __maybe_unused mx21_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx21_intctrl()  argument
521 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx21_intctrl()
524 static void __maybe_unused mx21_trigger(struct spi_imx_data *spi_imx)  in mx21_trigger()  argument
528 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx21_trigger()
530 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx21_trigger()
533 static int __maybe_unused mx21_config(struct spi_imx_data *spi_imx,  in mx21_config()  argument
537 	int cs = spi_imx->chipselect[config->cs];  in mx21_config()
538 	unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;  in mx21_config()
540 	reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, config->speed_hz, max) <<  in mx21_config()
553 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx21_config()
558 static int __maybe_unused mx21_rx_available(struct spi_imx_data *spi_imx)  in mx21_rx_available()  argument
560 	return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;  in mx21_rx_available()
563 static void __maybe_unused mx21_reset(struct spi_imx_data *spi_imx)  in mx21_reset()  argument
565 	writel(1, spi_imx->base + MXC_RESET);  in mx21_reset()
579 static void __maybe_unused mx1_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx1_intctrl()  argument
588 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx1_intctrl()
591 static void __maybe_unused mx1_trigger(struct spi_imx_data *spi_imx)  in mx1_trigger()  argument
595 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx1_trigger()
597 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx1_trigger()
600 static int __maybe_unused mx1_config(struct spi_imx_data *spi_imx,  in mx1_config()  argument
605 	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz) <<  in mx1_config()
614 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx1_config()
619 static int __maybe_unused mx1_rx_available(struct spi_imx_data *spi_imx)  in mx1_rx_available()  argument
621 	return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;  in mx1_rx_available()
624 static void __maybe_unused mx1_reset(struct spi_imx_data *spi_imx)  in mx1_reset()  argument
626 	writel(1, spi_imx->base + MXC_RESET);  in mx1_reset()
722 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_chipselect()  local
723 	int gpio = spi_imx->chipselect[spi->chip_select];  in spi_imx_chipselect()
733 static void spi_imx_push(struct spi_imx_data *spi_imx)  in spi_imx_push()  argument
735 	while (spi_imx->txfifo < spi_imx_get_fifosize(spi_imx)) {  in spi_imx_push()
736 		if (!spi_imx->count)  in spi_imx_push()
738 		spi_imx->tx(spi_imx);  in spi_imx_push()
739 		spi_imx->txfifo++;  in spi_imx_push()
742 	spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_push()
747 	struct spi_imx_data *spi_imx = dev_id;  in spi_imx_isr()  local
749 	while (spi_imx->devtype_data->rx_available(spi_imx)) {  in spi_imx_isr()
750 		spi_imx->rx(spi_imx);  in spi_imx_isr()
751 		spi_imx->txfifo--;  in spi_imx_isr()
754 	if (spi_imx->count) {  in spi_imx_isr()
755 		spi_imx_push(spi_imx);  in spi_imx_isr()
759 	if (spi_imx->txfifo) {  in spi_imx_isr()
763 		spi_imx->devtype_data->intctrl(  in spi_imx_isr()
764 				spi_imx, MXC_INT_RR);  in spi_imx_isr()
768 	spi_imx->devtype_data->intctrl(spi_imx, 0);  in spi_imx_isr()
769 	complete(&spi_imx->xfer_done);  in spi_imx_isr()
777 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_setupxfer()  local
792 		spi_imx->rx = spi_imx_buf_rx_u8;  in spi_imx_setupxfer()
793 		spi_imx->tx = spi_imx_buf_tx_u8;  in spi_imx_setupxfer()
795 		spi_imx->rx = spi_imx_buf_rx_u16;  in spi_imx_setupxfer()
796 		spi_imx->tx = spi_imx_buf_tx_u16;  in spi_imx_setupxfer()
798 		spi_imx->rx = spi_imx_buf_rx_u32;  in spi_imx_setupxfer()
799 		spi_imx->tx = spi_imx_buf_tx_u32;  in spi_imx_setupxfer()
802 	spi_imx->devtype_data->config(spi_imx, &config);  in spi_imx_setupxfer()
807 static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)  in spi_imx_sdma_exit()  argument
809 	struct spi_master *master = spi_imx->bitbang.master;  in spi_imx_sdma_exit()
821 	spi_imx->dma_is_inited = 0;  in spi_imx_sdma_exit()
824 static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,  in spi_imx_sdma_init()  argument
846 	slave_config.dst_maxburst = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
864 	slave_config.src_maxburst = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
871 	init_completion(&spi_imx->dma_rx_completion);  in spi_imx_sdma_init()
872 	init_completion(&spi_imx->dma_tx_completion);  in spi_imx_sdma_init()
875 	spi_imx->bitbang.master->flags = SPI_MASTER_MUST_RX |  in spi_imx_sdma_init()
877 	spi_imx->tx_wml = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
878 	spi_imx->rx_wml = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
879 	spi_imx->dma_is_inited = 1;  in spi_imx_sdma_init()
883 	spi_imx_sdma_exit(spi_imx);  in spi_imx_sdma_init()
889 	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;  in spi_imx_dma_rx_callback()  local
891 	complete(&spi_imx->dma_rx_completion);  in spi_imx_dma_rx_callback()
896 	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;  in spi_imx_dma_tx_callback()  local
898 	complete(&spi_imx->dma_tx_completion);  in spi_imx_dma_tx_callback()
901 static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,  in spi_imx_dma_transfer()  argument
909 	struct spi_master *master = spi_imx->bitbang.master;  in spi_imx_dma_transfer()
920 		desc_tx->callback_param = (void *)spi_imx;  in spi_imx_dma_transfer()
932 		desc_rx->callback_param = (void *)spi_imx;  in spi_imx_dma_transfer()
936 	reinit_completion(&spi_imx->dma_rx_completion);  in spi_imx_dma_transfer()
937 	reinit_completion(&spi_imx->dma_tx_completion);  in spi_imx_dma_transfer()
940 	spi_imx->dma_finished = 0;  in spi_imx_dma_transfer()
942 	dma = readl(spi_imx->base + MX51_ECSPI_DMA);  in spi_imx_dma_transfer()
945 	left = transfer->len % spi_imx->rxt_wml;  in spi_imx_dma_transfer()
948 				spi_imx->base + MX51_ECSPI_DMA);  in spi_imx_dma_transfer()
949 	spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_dma_transfer()
954 	timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,  in spi_imx_dma_transfer()
963 				&spi_imx->dma_rx_completion, IMX_DMA_TIMEOUT);  in spi_imx_dma_transfer()
968 			spi_imx->devtype_data->reset(spi_imx);  in spi_imx_dma_transfer()
972 		       spi_imx->rxt_wml << MX51_ECSPI_DMA_RXT_WML_OFFSET,  in spi_imx_dma_transfer()
973 		       spi_imx->base + MX51_ECSPI_DMA);  in spi_imx_dma_transfer()
976 	spi_imx->dma_finished = 1;  in spi_imx_dma_transfer()
977 	spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_dma_transfer()
996 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_pio_transfer()  local
998 	spi_imx->tx_buf = transfer->tx_buf;  in spi_imx_pio_transfer()
999 	spi_imx->rx_buf = transfer->rx_buf;  in spi_imx_pio_transfer()
1000 	spi_imx->count = transfer->len;  in spi_imx_pio_transfer()
1001 	spi_imx->txfifo = 0;  in spi_imx_pio_transfer()
1003 	reinit_completion(&spi_imx->xfer_done);  in spi_imx_pio_transfer()
1005 	spi_imx_push(spi_imx);  in spi_imx_pio_transfer()
1007 	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);  in spi_imx_pio_transfer()
1009 	wait_for_completion(&spi_imx->xfer_done);  in spi_imx_pio_transfer()
1018 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_transfer()  local
1020 	if (spi_imx->bitbang.master->can_dma &&  in spi_imx_transfer()
1021 	    spi_imx_can_dma(spi_imx->bitbang.master, spi, transfer)) {  in spi_imx_transfer()
1022 		spi_imx->usedma = true;  in spi_imx_transfer()
1023 		ret = spi_imx_dma_transfer(spi_imx, transfer);  in spi_imx_transfer()
1027 	spi_imx->usedma = false;  in spi_imx_transfer()
1034 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_setup()  local
1035 	int gpio = spi_imx->chipselect[spi->chip_select];  in spi_imx_setup()
1055 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_prepare_message()  local
1058 	ret = clk_enable(spi_imx->clk_per);  in spi_imx_prepare_message()
1062 	ret = clk_enable(spi_imx->clk_ipg);  in spi_imx_prepare_message()
1064 		clk_disable(spi_imx->clk_per);  in spi_imx_prepare_message()
1074 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_unprepare_message()  local
1076 	clk_disable(spi_imx->clk_ipg);  in spi_imx_unprepare_message()
1077 	clk_disable(spi_imx->clk_per);  in spi_imx_unprepare_message()
1089 	struct spi_imx_data *spi_imx;  in spi_imx_probe()  local
1117 	spi_imx = spi_master_get_devdata(master);  in spi_imx_probe()
1118 	spi_imx->bitbang.master = master;  in spi_imx_probe()
1125 		spi_imx->chipselect[i] = cs_gpio;  in spi_imx_probe()
1129 		ret = devm_gpio_request(&pdev->dev, spi_imx->chipselect[i],  in spi_imx_probe()
1137 	spi_imx->bitbang.chipselect = spi_imx_chipselect;  in spi_imx_probe()
1138 	spi_imx->bitbang.setup_transfer = spi_imx_setupxfer;  in spi_imx_probe()
1139 	spi_imx->bitbang.txrx_bufs = spi_imx_transfer;  in spi_imx_probe()
1140 	spi_imx->bitbang.master->setup = spi_imx_setup;  in spi_imx_probe()
1141 	spi_imx->bitbang.master->cleanup = spi_imx_cleanup;  in spi_imx_probe()
1142 	spi_imx->bitbang.master->prepare_message = spi_imx_prepare_message;  in spi_imx_probe()
1143 	spi_imx->bitbang.master->unprepare_message = spi_imx_unprepare_message;  in spi_imx_probe()
1144 	spi_imx->bitbang.master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;  in spi_imx_probe()
1146 	init_completion(&spi_imx->xfer_done);  in spi_imx_probe()
1148 	spi_imx->devtype_data = of_id ? of_id->data :  in spi_imx_probe()
1152 	spi_imx->base = devm_ioremap_resource(&pdev->dev, res);  in spi_imx_probe()
1153 	if (IS_ERR(spi_imx->base)) {  in spi_imx_probe()
1154 		ret = PTR_ERR(spi_imx->base);  in spi_imx_probe()
1165 			       dev_name(&pdev->dev), spi_imx);  in spi_imx_probe()
1171 	spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");  in spi_imx_probe()
1172 	if (IS_ERR(spi_imx->clk_ipg)) {  in spi_imx_probe()
1173 		ret = PTR_ERR(spi_imx->clk_ipg);  in spi_imx_probe()
1177 	spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");  in spi_imx_probe()
1178 	if (IS_ERR(spi_imx->clk_per)) {  in spi_imx_probe()
1179 		ret = PTR_ERR(spi_imx->clk_per);  in spi_imx_probe()
1183 	ret = clk_prepare_enable(spi_imx->clk_per);  in spi_imx_probe()
1187 	ret = clk_prepare_enable(spi_imx->clk_ipg);  in spi_imx_probe()
1191 	spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);  in spi_imx_probe()
1196 	if (spi_imx->devtype_data == &imx51_ecspi_devtype_data  in spi_imx_probe()
1197 	    && spi_imx_sdma_init(&pdev->dev, spi_imx, master, res))  in spi_imx_probe()
1200 	spi_imx->devtype_data->reset(spi_imx);  in spi_imx_probe()
1202 	spi_imx->devtype_data->intctrl(spi_imx, 0);  in spi_imx_probe()
1205 	ret = spi_bitbang_start(&spi_imx->bitbang);  in spi_imx_probe()
1213 	clk_disable(spi_imx->clk_ipg);  in spi_imx_probe()
1214 	clk_disable(spi_imx->clk_per);  in spi_imx_probe()
1218 	clk_disable_unprepare(spi_imx->clk_ipg);  in spi_imx_probe()
1220 	clk_disable_unprepare(spi_imx->clk_per);  in spi_imx_probe()
1230 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_remove()  local
1232 	spi_bitbang_stop(&spi_imx->bitbang);  in spi_imx_remove()
1234 	writel(0, spi_imx->base + MXC_CSPICTRL);  in spi_imx_remove()
1235 	clk_unprepare(spi_imx->clk_ipg);  in spi_imx_remove()
1236 	clk_unprepare(spi_imx->clk_per);  in spi_imx_remove()
1237 	spi_imx_sdma_exit(spi_imx);  in spi_imx_remove()