99 #define cafe_readl(cafe, addr)			readl((cafe)->mmio + CAFE_##addr)  argument
100 #define cafe_writel(cafe, datum, addr)		writel(datum, (cafe)->mmio + CAFE_##addr)  argument
104 	struct cafe_priv *cafe = mtd->priv;  in cafe_device_ready()  local
105 	int result = !!(cafe_readl(cafe, NAND_STATUS) & 0x40000000);  in cafe_device_ready()
106 	uint32_t irqs = cafe_readl(cafe, NAND_IRQ);  in cafe_device_ready()
108 	cafe_writel(cafe, irqs, NAND_IRQ);  in cafe_device_ready()
110 	cafe_dev_dbg(&cafe->pdev->dev, "NAND device is%s ready, IRQ %x (%x) (%x,%x)\n",  in cafe_device_ready()
111 		result?"":" not", irqs, cafe_readl(cafe, NAND_IRQ),  in cafe_device_ready()
112 		cafe_readl(cafe, GLOBAL_IRQ), cafe_readl(cafe, GLOBAL_IRQ_MASK));  in cafe_device_ready()
120 	struct cafe_priv *cafe = mtd->priv;  in cafe_write_buf()  local
123 		memcpy(cafe->dmabuf + cafe->datalen, buf, len);  in cafe_write_buf()
125 		memcpy_toio(cafe->mmio + CAFE_NAND_WRITE_DATA + cafe->datalen, buf, len);  in cafe_write_buf()
127 	cafe->datalen += len;  in cafe_write_buf()
129 	cafe_dev_dbg(&cafe->pdev->dev, "Copy 0x%x bytes to write buffer. datalen 0x%x\n",  in cafe_write_buf()
130 		len, cafe->datalen);  in cafe_write_buf()
135 	struct cafe_priv *cafe = mtd->priv;  in cafe_read_buf()  local
138 		memcpy(buf, cafe->dmabuf + cafe->datalen, len);  in cafe_read_buf()
140 		memcpy_fromio(buf, cafe->mmio + CAFE_NAND_READ_DATA + cafe->datalen, len);  in cafe_read_buf()
142 	cafe_dev_dbg(&cafe->pdev->dev, "Copy 0x%x bytes from position 0x%x in read buffer.\n",  in cafe_read_buf()
143 		  len, cafe->datalen);  in cafe_read_buf()
144 	cafe->datalen += len;  in cafe_read_buf()
149 	struct cafe_priv *cafe = mtd->priv;  in cafe_read_byte()  local
153 	cafe_dev_dbg(&cafe->pdev->dev, "Read %02x\n", d);  in cafe_read_byte()
161 	struct cafe_priv *cafe = mtd->priv;  in cafe_nand_cmdfunc()  local
166 	cafe_dev_dbg(&cafe->pdev->dev, "cmdfunc %02x, 0x%x, 0x%x\n",  in cafe_nand_cmdfunc()
171 		cafe_writel(cafe, cafe->ctl2 | 0x100 | command, NAND_CTRL2);  in cafe_nand_cmdfunc()
172 		ctl1 = cafe->ctl1;  in cafe_nand_cmdfunc()
173 		cafe->ctl2 &= ~(1<<30);  in cafe_nand_cmdfunc()
174 		cafe_dev_dbg(&cafe->pdev->dev, "Continue command, ctl1 %08x, #data %d\n",  in cafe_nand_cmdfunc()
175 			  cafe->ctl1, cafe->nr_data);  in cafe_nand_cmdfunc()
179 	cafe_writel(cafe, 0, NAND_CTRL2);  in cafe_nand_cmdfunc()
192 		cafe_writel(cafe, column, NAND_ADDR1);  in cafe_nand_cmdfunc()
197 		cafe_writel(cafe, page_addr & 0xffff, NAND_ADDR1);  in cafe_nand_cmdfunc()
200 		cafe_writel(cafe, page_addr, NAND_ADDR2);  in cafe_nand_cmdfunc()
206 	cafe->data_pos = cafe->datalen = 0;  in cafe_nand_cmdfunc()
209 	ctl1 = 0x80000000 | command | (cafe->ctl1 & CTRL1_CHIPSELECT);  in cafe_nand_cmdfunc()
215 		cafe->datalen = 4;  in cafe_nand_cmdfunc()
222 		cafe->datalen = mtd->writesize + mtd->oobsize - column;  in cafe_nand_cmdfunc()
233 		cafe->ctl1 = ctl1;  in cafe_nand_cmdfunc()
234 		cafe_dev_dbg(&cafe->pdev->dev, "Setup for delayed command, ctl1 %08x, dlen %x\n",  in cafe_nand_cmdfunc()
235 			  cafe->ctl1, cafe->datalen);  in cafe_nand_cmdfunc()
240 		cafe_writel(cafe, cafe->ctl2 | 0x100 | NAND_CMD_RNDOUTSTART, NAND_CTRL2);  in cafe_nand_cmdfunc()
242 		cafe_writel(cafe, cafe->ctl2 | 0x100 | NAND_CMD_READSTART, NAND_CTRL2);  in cafe_nand_cmdfunc()
245 	cafe_dev_dbg(&cafe->pdev->dev, "dlen %x, ctl1 %x, ctl2 %x\n",  in cafe_nand_cmdfunc()
246 		cafe->datalen, ctl1, cafe_readl(cafe, NAND_CTRL2));  in cafe_nand_cmdfunc()
249 	cafe_writel(cafe, cafe->datalen, NAND_DATA_LEN);  in cafe_nand_cmdfunc()
250 	cafe_writel(cafe, 0x90000000, NAND_IRQ);  in cafe_nand_cmdfunc()
252 		uint32_t dmactl = 0xc0000000 + cafe->datalen;  in cafe_nand_cmdfunc()
261 		cafe_writel(cafe, dmactl, NAND_DMA_CTRL);  in cafe_nand_cmdfunc()
263 	cafe->datalen = 0;  in cafe_nand_cmdfunc()
269 			printk("Register %x: %08x\n", i, readl(cafe->mmio + i));  in cafe_nand_cmdfunc()
272 	cafe_writel(cafe, ctl1, NAND_CTRL1);  in cafe_nand_cmdfunc()
282 			irqs = cafe_readl(cafe, NAND_IRQ);  in cafe_nand_cmdfunc()
287 				cafe_dev_dbg(&cafe->pdev->dev, "Wait for ready, IRQ %x\n", irqs);  in cafe_nand_cmdfunc()
290 		cafe_writel(cafe, doneint, NAND_IRQ);  in cafe_nand_cmdfunc()
291 		cafe_dev_dbg(&cafe->pdev->dev, "Command %x completed after %d usec, irqs %x (%x)\n",  in cafe_nand_cmdfunc()
292 			     command, 500000-c, irqs, cafe_readl(cafe, NAND_IRQ));  in cafe_nand_cmdfunc()
295 	WARN_ON(cafe->ctl2 & (1<<30));  in cafe_nand_cmdfunc()
307 		cafe_writel(cafe, cafe->ctl2, NAND_CTRL2);  in cafe_nand_cmdfunc()
311 	cafe_writel(cafe, cafe->ctl2, NAND_CTRL2);  in cafe_nand_cmdfunc()
316 	struct cafe_priv *cafe = mtd->priv;  in cafe_select_chip()  local
318 	cafe_dev_dbg(&cafe->pdev->dev, "select_chip %d\n", chipnr);  in cafe_select_chip()
323 		cafe->ctl1 |= CTRL1_CHIPSELECT;  in cafe_select_chip()
325 		cafe->ctl1 &= ~CTRL1_CHIPSELECT;  in cafe_select_chip()
331 	struct cafe_priv *cafe = mtd->priv;  in cafe_nand_interrupt()  local
332 	uint32_t irqs = cafe_readl(cafe, NAND_IRQ);  in cafe_nand_interrupt()
333 	cafe_writel(cafe, irqs & ~0x90000000, NAND_IRQ);  in cafe_nand_interrupt()
337 	cafe_dev_dbg(&cafe->pdev->dev, "irq, bits %x (%x)\n", irqs, cafe_readl(cafe, NAND_IRQ));  in cafe_nand_interrupt()
380 	struct cafe_priv *cafe = mtd->priv;  in cafe_nand_read_page()  local
383 	cafe_dev_dbg(&cafe->pdev->dev, "ECC result %08x SYN1,2 %08x\n",  in cafe_nand_read_page()
384 		     cafe_readl(cafe, NAND_ECC_RESULT),  in cafe_nand_read_page()
385 		     cafe_readl(cafe, NAND_ECC_SYN01));  in cafe_nand_read_page()
390 	if (checkecc && cafe_readl(cafe, NAND_ECC_RESULT) & (1<<18)) {  in cafe_nand_read_page()
397 			uint32_t tmp = cafe_readl(cafe, NAND_ECC_SYN01 + (i*2));  in cafe_nand_read_page()
398 			syn[i] = cafe->rs->index_of[tmp & 0xfff];  in cafe_nand_read_page()
399 			syn[i+1] = cafe->rs->index_of[(tmp >> 16) & 0xfff];  in cafe_nand_read_page()
402 		n = decode_rs16(cafe->rs, NULL, NULL, 1367, syn, 0, pos, 0,  in cafe_nand_read_page()
440 			dev_dbg(&cafe->pdev->dev, "Failed to correct ECC at %08x\n",  in cafe_nand_read_page()
441 				cafe_readl(cafe, NAND_ADDR2) * 2048);  in cafe_nand_read_page()
443 				printk("Register %x: %08x\n", i, readl(cafe->mmio + i));  in cafe_nand_read_page()
446 			dev_dbg(&cafe->pdev->dev, "Corrected %d symbol errors\n", n);  in cafe_nand_read_page()
521 	struct cafe_priv *cafe = mtd->priv;  in cafe_nand_write_page_lowlevel()  local
527 	cafe->ctl2 |= (1<<30);  in cafe_nand_write_page_lowlevel()
583 	struct cafe_priv *cafe;  in cafe_nand_probe()  local
603 	cafe = (void *)(&mtd[1]);  in cafe_nand_probe()
606 	mtd->priv = cafe;  in cafe_nand_probe()
609 	cafe->pdev = pdev;  in cafe_nand_probe()
610 	cafe->mmio = pci_iomap(pdev, 0, 0);  in cafe_nand_probe()
611 	if (!cafe->mmio) {  in cafe_nand_probe()
617 	cafe->rs = init_rs_non_canonical(12, &cafe_mul, 0, 1, 8);  in cafe_nand_probe()
618 	if (!cafe->rs) {  in cafe_nand_probe()
623 	cafe->nand.cmdfunc = cafe_nand_cmdfunc;  in cafe_nand_probe()
624 	cafe->nand.dev_ready = cafe_device_ready;  in cafe_nand_probe()
625 	cafe->nand.read_byte = cafe_read_byte;  in cafe_nand_probe()
626 	cafe->nand.read_buf = cafe_read_buf;  in cafe_nand_probe()
627 	cafe->nand.write_buf = cafe_write_buf;  in cafe_nand_probe()
628 	cafe->nand.select_chip = cafe_select_chip;  in cafe_nand_probe()
630 	cafe->nand.chip_delay = 0;  in cafe_nand_probe()
633 	cafe->nand.bbt_options = NAND_BBT_USE_FLASH;  in cafe_nand_probe()
634 	cafe->nand.options = NAND_OWN_BUFFERS;  in cafe_nand_probe()
637 		cafe->nand.options |= NAND_SKIP_BBTSCAN;  in cafe_nand_probe()
638 		cafe->nand.block_bad = cafe_nand_block_bad;  in cafe_nand_probe()
642 …dev_warn(&cafe->pdev->dev, "%d timing register values ignored; precisely three are required\n", nu…  in cafe_nand_probe()
646 		cafe_dev_dbg(&cafe->pdev->dev, "Using provided timings (%08x %08x %08x)\n",  in cafe_nand_probe()
649 		timing[0] = cafe_readl(cafe, NAND_TIMING1);  in cafe_nand_probe()
650 		timing[1] = cafe_readl(cafe, NAND_TIMING2);  in cafe_nand_probe()
651 		timing[2] = cafe_readl(cafe, NAND_TIMING3);  in cafe_nand_probe()
654 			cafe_dev_dbg(&cafe->pdev->dev, "Timing registers already set (%08x %08x %08x)\n",  in cafe_nand_probe()
657 			dev_warn(&cafe->pdev->dev, "Timing registers unset; using most conservative defaults\n");  in cafe_nand_probe()
663 	cafe_writel(cafe, 1, NAND_RESET);  in cafe_nand_probe()
664 	cafe_writel(cafe, 0, NAND_RESET);  in cafe_nand_probe()
666 	cafe_writel(cafe, timing[0], NAND_TIMING1);  in cafe_nand_probe()
667 	cafe_writel(cafe, timing[1], NAND_TIMING2);  in cafe_nand_probe()
668 	cafe_writel(cafe, timing[2], NAND_TIMING3);  in cafe_nand_probe()
670 	cafe_writel(cafe, 0xffffffff, NAND_IRQ_MASK);  in cafe_nand_probe()
679 	ctrl = cafe_readl(cafe, GLOBAL_CTRL);  in cafe_nand_probe()
682 	cafe_writel(cafe, ctrl | 0x05, GLOBAL_CTRL);  in cafe_nand_probe()
683 	cafe_writel(cafe, ctrl | 0x0a, GLOBAL_CTRL);  in cafe_nand_probe()
684 	cafe_writel(cafe, 0, NAND_DMA_CTRL);  in cafe_nand_probe()
686 	cafe_writel(cafe, 0x7006, GLOBAL_CTRL);  in cafe_nand_probe()
687 	cafe_writel(cafe, 0x700a, GLOBAL_CTRL);  in cafe_nand_probe()
690 	cafe_writel(cafe, 0x80000007, GLOBAL_IRQ_MASK);  in cafe_nand_probe()
691 	cafe_dev_dbg(&cafe->pdev->dev, "Control %x, IRQ mask %x\n",  in cafe_nand_probe()
692 		cafe_readl(cafe, GLOBAL_CTRL),  in cafe_nand_probe()
693 		cafe_readl(cafe, GLOBAL_IRQ_MASK));  in cafe_nand_probe()
705 	cafe->dmabuf = dma_alloc_coherent(&cafe->pdev->dev,  in cafe_nand_probe()
708 				&cafe->dmaaddr, GFP_KERNEL);  in cafe_nand_probe()
709 	if (!cafe->dmabuf) {  in cafe_nand_probe()
713 	cafe->nand.buffers = nbuf = (void *)cafe->dmabuf + 2112;  in cafe_nand_probe()
716 	cafe_writel(cafe, cafe->dmaaddr & 0xffffffff, NAND_DMA_ADDR0);  in cafe_nand_probe()
717 	if (sizeof(cafe->dmaaddr) > 4)  in cafe_nand_probe()
719 		cafe_writel(cafe, (cafe->dmaaddr >> 16) >> 16, NAND_DMA_ADDR1);  in cafe_nand_probe()
721 		cafe_writel(cafe, 0, NAND_DMA_ADDR1);  in cafe_nand_probe()
723 	cafe_dev_dbg(&cafe->pdev->dev, "Set DMA address to %x (virt %p)\n",  in cafe_nand_probe()
724 		cafe_readl(cafe, NAND_DMA_ADDR0), cafe->dmabuf);  in cafe_nand_probe()
734 	cafe->ctl2 = 1<<27; /* Reed-Solomon ECC */  in cafe_nand_probe()
736 		cafe->ctl2 |= 1<<29; /* 2KiB page size */  in cafe_nand_probe()
740 		cafe->nand.ecc.layout = &cafe_oobinfo_2048;  in cafe_nand_probe()
741 		cafe->nand.bbt_td = &cafe_bbt_main_descr_2048;  in cafe_nand_probe()
742 		cafe->nand.bbt_md = &cafe_bbt_mirror_descr_2048;  in cafe_nand_probe()
744 		cafe->nand.ecc.layout = &cafe_oobinfo_512;  in cafe_nand_probe()
745 		cafe->nand.bbt_td = &cafe_bbt_main_descr_512;  in cafe_nand_probe()
746 		cafe->nand.bbt_md = &cafe_bbt_mirror_descr_512;  in cafe_nand_probe()
752 	cafe->nand.ecc.mode = NAND_ECC_HW_SYNDROME;  in cafe_nand_probe()
753 	cafe->nand.ecc.size = mtd->writesize;  in cafe_nand_probe()
754 	cafe->nand.ecc.bytes = 14;  in cafe_nand_probe()
755 	cafe->nand.ecc.strength = 4;  in cafe_nand_probe()
756 	cafe->nand.ecc.hwctl  = (void *)cafe_nand_bug;  in cafe_nand_probe()
757 	cafe->nand.ecc.calculate = (void *)cafe_nand_bug;  in cafe_nand_probe()
758 	cafe->nand.ecc.correct  = (void *)cafe_nand_bug;  in cafe_nand_probe()
759 	cafe->nand.ecc.write_page = cafe_nand_write_page_lowlevel;  in cafe_nand_probe()
760 	cafe->nand.ecc.write_oob = cafe_nand_write_oob;  in cafe_nand_probe()
761 	cafe->nand.ecc.read_page = cafe_nand_read_page;  in cafe_nand_probe()
762 	cafe->nand.ecc.read_oob = cafe_nand_read_oob;  in cafe_nand_probe()
776 	dma_free_coherent(&cafe->pdev->dev,  in cafe_nand_probe()
779 			cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_probe()
782 	cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK);  in cafe_nand_probe()
785 	pci_iounmap(pdev, cafe->mmio);  in cafe_nand_probe()
795 	struct cafe_priv *cafe = mtd->priv;  in cafe_nand_remove()  local
798 	cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK);  in cafe_nand_remove()
801 	free_rs(cafe->rs);  in cafe_nand_remove()
802 	pci_iounmap(pdev, cafe->mmio);  in cafe_nand_remove()
803 	dma_free_coherent(&cafe->pdev->dev,  in cafe_nand_remove()
806 			cafe->dmabuf, cafe->dmaaddr);  in cafe_nand_remove()
822 	struct cafe_priv *cafe = mtd->priv;  in cafe_nand_resume()  local
825 	cafe_writel(cafe, 1, NAND_RESET);  in cafe_nand_resume()
826 	cafe_writel(cafe, 0, NAND_RESET);  in cafe_nand_resume()
827 	cafe_writel(cafe, 0xffffffff, NAND_IRQ_MASK);  in cafe_nand_resume()
830 	cafe_writel(cafe, timing[0], NAND_TIMING1);  in cafe_nand_resume()
831 	cafe_writel(cafe, timing[1], NAND_TIMING2);  in cafe_nand_resume()
832 	cafe_writel(cafe, timing[2], NAND_TIMING3);  in cafe_nand_resume()
835 	ctrl = cafe_readl(cafe, GLOBAL_CTRL);  in cafe_nand_resume()
838 	cafe_writel(cafe, ctrl | 0x05, GLOBAL_CTRL);  in cafe_nand_resume()
839 	cafe_writel(cafe, ctrl | 0x0a, GLOBAL_CTRL);  in cafe_nand_resume()
840 	cafe_writel(cafe, 0, NAND_DMA_CTRL);  in cafe_nand_resume()
841 	cafe_writel(cafe, 0x7006, GLOBAL_CTRL);  in cafe_nand_resume()
842 	cafe_writel(cafe, 0x700a, GLOBAL_CTRL);  in cafe_nand_resume()
845 	cafe_writel(cafe, cafe->dmaaddr & 0xffffffff, NAND_DMA_ADDR0);  in cafe_nand_resume()
846 	if (sizeof(cafe->dmaaddr) > 4)  in cafe_nand_resume()
848 		cafe_writel(cafe, (cafe->dmaaddr >> 16) >> 16, NAND_DMA_ADDR1);  in cafe_nand_resume()
850 		cafe_writel(cafe, 0, NAND_DMA_ADDR1);  in cafe_nand_resume()
853 	cafe_writel(cafe, 0x80000007, GLOBAL_IRQ_MASK);  in cafe_nand_resume()