133 static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx)		\
135 	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);	\
137 	if (spi_imx->rx_buf) {						\
138 		*(type *)spi_imx->rx_buf = val;				\
139 		spi_imx->rx_buf += sizeof(type);			\
144 static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx)		\
148 	if (spi_imx->tx_buf) {						\
149 		val = *(type *)spi_imx->tx_buf;				\
150 		spi_imx->tx_buf += sizeof(type);			\
153 	spi_imx->count -= sizeof(type);					\
155 	writel(val, spi_imx->base + MXC_CSPITXDATA);			\
202 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_can_dma()  local
204 	if (spi_imx->dma_is_inited  in spi_imx_can_dma()
205 	    && transfer->len > spi_imx->rx_wml * sizeof(u32)  in spi_imx_can_dma()
206 	    && transfer->len > spi_imx->tx_wml * sizeof(u32))  in spi_imx_can_dma()
285 static void __maybe_unused mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx51_ecspi_intctrl()  argument
295 	writel(val, spi_imx->base + MX51_ECSPI_INT);  in mx51_ecspi_intctrl()
298 static void __maybe_unused mx51_ecspi_trigger(struct spi_imx_data *spi_imx)  in mx51_ecspi_trigger()  argument
300 	u32 reg = readl(spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_trigger()
302 	if (!spi_imx->usedma)  in mx51_ecspi_trigger()
304 	else if (!spi_imx->dma_finished)  in mx51_ecspi_trigger()
308 	writel(reg, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_trigger()
311 static int __maybe_unused mx51_ecspi_config(struct spi_imx_data *spi_imx,  in mx51_ecspi_config()  argument
328 	ctrl |= mx51_ecspi_clkdiv(spi_imx->spi_clk, config->speed_hz, &clk);  in mx51_ecspi_config()
347 	writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);  in mx51_ecspi_config()
348 	writel(cfg, spi_imx->base + MX51_ECSPI_CONFIG);  in mx51_ecspi_config()
371 	if (spi_imx->dma_is_inited) {  in mx51_ecspi_config()
372 		dma = readl(spi_imx->base + MX51_ECSPI_DMA);  in mx51_ecspi_config()
374 		spi_imx->rxt_wml = spi_imx_get_fifosize(spi_imx) / 2;  in mx51_ecspi_config()
375 		rx_wml_cfg = spi_imx->rx_wml << MX51_ECSPI_DMA_RX_WML_OFFSET;  in mx51_ecspi_config()
376 		tx_wml_cfg = spi_imx->tx_wml << MX51_ECSPI_DMA_TX_WML_OFFSET;  in mx51_ecspi_config()
377 		rxt_wml_cfg = spi_imx->rxt_wml << MX51_ECSPI_DMA_RXT_WML_OFFSET;  in mx51_ecspi_config()
386 		writel(dma, spi_imx->base + MX51_ECSPI_DMA);  in mx51_ecspi_config()
392 static int __maybe_unused mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)  in mx51_ecspi_rx_available()  argument
394 	return readl(spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;  in mx51_ecspi_rx_available()
397 static void __maybe_unused mx51_ecspi_reset(struct spi_imx_data *spi_imx)  in mx51_ecspi_reset()  argument
400 	while (mx51_ecspi_rx_available(spi_imx))  in mx51_ecspi_reset()
401 		readl(spi_imx->base + MXC_CSPIRXDATA);  in mx51_ecspi_reset()
427 static void __maybe_unused mx31_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx31_intctrl()  argument
436 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx31_intctrl()
439 static void __maybe_unused mx31_trigger(struct spi_imx_data *spi_imx)  in mx31_trigger()  argument
443 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx31_trigger()
445 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx31_trigger()
448 static int __maybe_unused mx31_config(struct spi_imx_data *spi_imx,  in mx31_config()  argument
452 	int cs = spi_imx->chipselect[config->cs];  in mx31_config()
454 	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz) <<  in mx31_config()
457 	if (is_imx35_cspi(spi_imx)) {  in mx31_config()
472 			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :  in mx31_config()
475 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx31_config()
480 static int __maybe_unused mx31_rx_available(struct spi_imx_data *spi_imx)  in mx31_rx_available()  argument
482 	return readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;  in mx31_rx_available()
485 static void __maybe_unused mx31_reset(struct spi_imx_data *spi_imx)  in mx31_reset()  argument
488 	while (readl(spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)  in mx31_reset()
489 		readl(spi_imx->base + MXC_CSPIRXDATA);  in mx31_reset()
505 static void __maybe_unused mx21_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx21_intctrl()  argument
514 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx21_intctrl()
517 static void __maybe_unused mx21_trigger(struct spi_imx_data *spi_imx)  in mx21_trigger()  argument
521 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx21_trigger()
523 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx21_trigger()
526 static int __maybe_unused mx21_config(struct spi_imx_data *spi_imx,  in mx21_config()  argument
530 	int cs = spi_imx->chipselect[config->cs];  in mx21_config()
531 	unsigned int max = is_imx27_cspi(spi_imx) ? 16 : 18;  in mx21_config()
533 	reg |= spi_imx_clkdiv_1(spi_imx->spi_clk, config->speed_hz, max) <<  in mx21_config()
546 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx21_config()
551 static int __maybe_unused mx21_rx_available(struct spi_imx_data *spi_imx)  in mx21_rx_available()  argument
553 	return readl(spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;  in mx21_rx_available()
556 static void __maybe_unused mx21_reset(struct spi_imx_data *spi_imx)  in mx21_reset()  argument
558 	writel(1, spi_imx->base + MXC_RESET);  in mx21_reset()
572 static void __maybe_unused mx1_intctrl(struct spi_imx_data *spi_imx, int enable)  in mx1_intctrl()  argument
581 	writel(val, spi_imx->base + MXC_CSPIINT);  in mx1_intctrl()
584 static void __maybe_unused mx1_trigger(struct spi_imx_data *spi_imx)  in mx1_trigger()  argument
588 	reg = readl(spi_imx->base + MXC_CSPICTRL);  in mx1_trigger()
590 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx1_trigger()
593 static int __maybe_unused mx1_config(struct spi_imx_data *spi_imx,  in mx1_config()  argument
598 	reg |= spi_imx_clkdiv_2(spi_imx->spi_clk, config->speed_hz) <<  in mx1_config()
607 	writel(reg, spi_imx->base + MXC_CSPICTRL);  in mx1_config()
612 static int __maybe_unused mx1_rx_available(struct spi_imx_data *spi_imx)  in mx1_rx_available()  argument
614 	return readl(spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;  in mx1_rx_available()
617 static void __maybe_unused mx1_reset(struct spi_imx_data *spi_imx)  in mx1_reset()  argument
619 	writel(1, spi_imx->base + MXC_RESET);  in mx1_reset()
715 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_chipselect()  local
716 	int gpio = spi_imx->chipselect[spi->chip_select];  in spi_imx_chipselect()
726 static void spi_imx_push(struct spi_imx_data *spi_imx)  in spi_imx_push()  argument
728 	while (spi_imx->txfifo < spi_imx_get_fifosize(spi_imx)) {  in spi_imx_push()
729 		if (!spi_imx->count)  in spi_imx_push()
731 		spi_imx->tx(spi_imx);  in spi_imx_push()
732 		spi_imx->txfifo++;  in spi_imx_push()
735 	spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_push()
740 	struct spi_imx_data *spi_imx = dev_id;  in spi_imx_isr()  local
742 	while (spi_imx->devtype_data->rx_available(spi_imx)) {  in spi_imx_isr()
743 		spi_imx->rx(spi_imx);  in spi_imx_isr()
744 		spi_imx->txfifo--;  in spi_imx_isr()
747 	if (spi_imx->count) {  in spi_imx_isr()
748 		spi_imx_push(spi_imx);  in spi_imx_isr()
752 	if (spi_imx->txfifo) {  in spi_imx_isr()
756 		spi_imx->devtype_data->intctrl(  in spi_imx_isr()
757 				spi_imx, MXC_INT_RR);  in spi_imx_isr()
761 	spi_imx->devtype_data->intctrl(spi_imx, 0);  in spi_imx_isr()
762 	complete(&spi_imx->xfer_done);  in spi_imx_isr()
770 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_setupxfer()  local
785 		spi_imx->rx = spi_imx_buf_rx_u8;  in spi_imx_setupxfer()
786 		spi_imx->tx = spi_imx_buf_tx_u8;  in spi_imx_setupxfer()
788 		spi_imx->rx = spi_imx_buf_rx_u16;  in spi_imx_setupxfer()
789 		spi_imx->tx = spi_imx_buf_tx_u16;  in spi_imx_setupxfer()
791 		spi_imx->rx = spi_imx_buf_rx_u32;  in spi_imx_setupxfer()
792 		spi_imx->tx = spi_imx_buf_tx_u32;  in spi_imx_setupxfer()
795 	spi_imx->devtype_data->config(spi_imx, &config);  in spi_imx_setupxfer()
800 static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)  in spi_imx_sdma_exit()  argument
802 	struct spi_master *master = spi_imx->bitbang.master;  in spi_imx_sdma_exit()
814 	spi_imx->dma_is_inited = 0;  in spi_imx_sdma_exit()
817 static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,  in spi_imx_sdma_init()  argument
839 	slave_config.dst_maxburst = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
857 	slave_config.src_maxburst = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
864 	init_completion(&spi_imx->dma_rx_completion);  in spi_imx_sdma_init()
865 	init_completion(&spi_imx->dma_tx_completion);  in spi_imx_sdma_init()
868 	spi_imx->bitbang.master->flags = SPI_MASTER_MUST_RX |  in spi_imx_sdma_init()
870 	spi_imx->tx_wml = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
871 	spi_imx->rx_wml = spi_imx_get_fifosize(spi_imx) / 2;  in spi_imx_sdma_init()
872 	spi_imx->dma_is_inited = 1;  in spi_imx_sdma_init()
876 	spi_imx_sdma_exit(spi_imx);  in spi_imx_sdma_init()
882 	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;  in spi_imx_dma_rx_callback()  local
884 	complete(&spi_imx->dma_rx_completion);  in spi_imx_dma_rx_callback()
889 	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;  in spi_imx_dma_tx_callback()  local
891 	complete(&spi_imx->dma_tx_completion);  in spi_imx_dma_tx_callback()
894 static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,  in spi_imx_dma_transfer()  argument
902 	struct spi_master *master = spi_imx->bitbang.master;  in spi_imx_dma_transfer()
913 		desc_tx->callback_param = (void *)spi_imx;  in spi_imx_dma_transfer()
925 		desc_rx->callback_param = (void *)spi_imx;  in spi_imx_dma_transfer()
929 	reinit_completion(&spi_imx->dma_rx_completion);  in spi_imx_dma_transfer()
930 	reinit_completion(&spi_imx->dma_tx_completion);  in spi_imx_dma_transfer()
933 	spi_imx->dma_finished = 0;  in spi_imx_dma_transfer()
935 	dma = readl(spi_imx->base + MX51_ECSPI_DMA);  in spi_imx_dma_transfer()
938 	left = transfer->len % spi_imx->rxt_wml;  in spi_imx_dma_transfer()
941 				spi_imx->base + MX51_ECSPI_DMA);  in spi_imx_dma_transfer()
942 	spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_dma_transfer()
947 	timeout = wait_for_completion_timeout(&spi_imx->dma_tx_completion,  in spi_imx_dma_transfer()
956 				&spi_imx->dma_rx_completion, IMX_DMA_TIMEOUT);  in spi_imx_dma_transfer()
961 			spi_imx->devtype_data->reset(spi_imx);  in spi_imx_dma_transfer()
965 		       spi_imx->rxt_wml << MX51_ECSPI_DMA_RXT_WML_OFFSET,  in spi_imx_dma_transfer()
966 		       spi_imx->base + MX51_ECSPI_DMA);  in spi_imx_dma_transfer()
969 	spi_imx->dma_finished = 1;  in spi_imx_dma_transfer()
970 	spi_imx->devtype_data->trigger(spi_imx);  in spi_imx_dma_transfer()
989 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_pio_transfer()  local
991 	spi_imx->tx_buf = transfer->tx_buf;  in spi_imx_pio_transfer()
992 	spi_imx->rx_buf = transfer->rx_buf;  in spi_imx_pio_transfer()
993 	spi_imx->count = transfer->len;  in spi_imx_pio_transfer()
994 	spi_imx->txfifo = 0;  in spi_imx_pio_transfer()
996 	reinit_completion(&spi_imx->xfer_done);  in spi_imx_pio_transfer()
998 	spi_imx_push(spi_imx);  in spi_imx_pio_transfer()
1000 	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);  in spi_imx_pio_transfer()
1002 	wait_for_completion(&spi_imx->xfer_done);  in spi_imx_pio_transfer()
1011 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_transfer()  local
1013 	if (spi_imx->bitbang.master->can_dma &&  in spi_imx_transfer()
1014 	    spi_imx_can_dma(spi_imx->bitbang.master, spi, transfer)) {  in spi_imx_transfer()
1015 		spi_imx->usedma = true;  in spi_imx_transfer()
1016 		ret = spi_imx_dma_transfer(spi_imx, transfer);  in spi_imx_transfer()
1020 	spi_imx->usedma = false;  in spi_imx_transfer()
1027 	struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);  in spi_imx_setup()  local
1028 	int gpio = spi_imx->chipselect[spi->chip_select];  in spi_imx_setup()
1048 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_prepare_message()  local
1051 	ret = clk_enable(spi_imx->clk_per);  in spi_imx_prepare_message()
1055 	ret = clk_enable(spi_imx->clk_ipg);  in spi_imx_prepare_message()
1057 		clk_disable(spi_imx->clk_per);  in spi_imx_prepare_message()
1067 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_unprepare_message()  local
1069 	clk_disable(spi_imx->clk_ipg);  in spi_imx_unprepare_message()
1070 	clk_disable(spi_imx->clk_per);  in spi_imx_unprepare_message()
1082 	struct spi_imx_data *spi_imx;  in spi_imx_probe()  local
1110 	spi_imx = spi_master_get_devdata(master);  in spi_imx_probe()
1111 	spi_imx->bitbang.master = master;  in spi_imx_probe()
1118 		spi_imx->chipselect[i] = cs_gpio;  in spi_imx_probe()
1122 		ret = devm_gpio_request(&pdev->dev, spi_imx->chipselect[i],  in spi_imx_probe()
1130 	spi_imx->bitbang.chipselect = spi_imx_chipselect;  in spi_imx_probe()
1131 	spi_imx->bitbang.setup_transfer = spi_imx_setupxfer;  in spi_imx_probe()
1132 	spi_imx->bitbang.txrx_bufs = spi_imx_transfer;  in spi_imx_probe()
1133 	spi_imx->bitbang.master->setup = spi_imx_setup;  in spi_imx_probe()
1134 	spi_imx->bitbang.master->cleanup = spi_imx_cleanup;  in spi_imx_probe()
1135 	spi_imx->bitbang.master->prepare_message = spi_imx_prepare_message;  in spi_imx_probe()
1136 	spi_imx->bitbang.master->unprepare_message = spi_imx_unprepare_message;  in spi_imx_probe()
1137 	spi_imx->bitbang.master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;  in spi_imx_probe()
1139 	init_completion(&spi_imx->xfer_done);  in spi_imx_probe()
1141 	spi_imx->devtype_data = of_id ? of_id->data :  in spi_imx_probe()
1145 	spi_imx->base = devm_ioremap_resource(&pdev->dev, res);  in spi_imx_probe()
1146 	if (IS_ERR(spi_imx->base)) {  in spi_imx_probe()
1147 		ret = PTR_ERR(spi_imx->base);  in spi_imx_probe()
1158 			       dev_name(&pdev->dev), spi_imx);  in spi_imx_probe()
1164 	spi_imx->clk_ipg = devm_clk_get(&pdev->dev, "ipg");  in spi_imx_probe()
1165 	if (IS_ERR(spi_imx->clk_ipg)) {  in spi_imx_probe()
1166 		ret = PTR_ERR(spi_imx->clk_ipg);  in spi_imx_probe()
1170 	spi_imx->clk_per = devm_clk_get(&pdev->dev, "per");  in spi_imx_probe()
1171 	if (IS_ERR(spi_imx->clk_per)) {  in spi_imx_probe()
1172 		ret = PTR_ERR(spi_imx->clk_per);  in spi_imx_probe()
1176 	ret = clk_prepare_enable(spi_imx->clk_per);  in spi_imx_probe()
1180 	ret = clk_prepare_enable(spi_imx->clk_ipg);  in spi_imx_probe()
1184 	spi_imx->spi_clk = clk_get_rate(spi_imx->clk_per);  in spi_imx_probe()
1189 	if (spi_imx->devtype_data == &imx51_ecspi_devtype_data  in spi_imx_probe()
1190 	    && spi_imx_sdma_init(&pdev->dev, spi_imx, master, res))  in spi_imx_probe()
1193 	spi_imx->devtype_data->reset(spi_imx);  in spi_imx_probe()
1195 	spi_imx->devtype_data->intctrl(spi_imx, 0);  in spi_imx_probe()
1198 	ret = spi_bitbang_start(&spi_imx->bitbang);  in spi_imx_probe()
1206 	clk_disable(spi_imx->clk_ipg);  in spi_imx_probe()
1207 	clk_disable(spi_imx->clk_per);  in spi_imx_probe()
1211 	clk_disable_unprepare(spi_imx->clk_ipg);  in spi_imx_probe()
1213 	clk_disable_unprepare(spi_imx->clk_per);  in spi_imx_probe()
1223 	struct spi_imx_data *spi_imx = spi_master_get_devdata(master);  in spi_imx_remove()  local
1225 	spi_bitbang_stop(&spi_imx->bitbang);  in spi_imx_remove()
1227 	writel(0, spi_imx->base + MXC_CSPICTRL);  in spi_imx_remove()
1228 	clk_unprepare(spi_imx->clk_ipg);  in spi_imx_remove()
1229 	clk_unprepare(spi_imx->clk_per);  in spi_imx_remove()
1230 	spi_imx_sdma_exit(spi_imx);  in spi_imx_remove()