1/*
2 * Core driver for the High Speed UART DMA
3 *
4 * Copyright (C) 2015 Intel Corporation
5 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
6 *
7 * Partially based on the bits found in drivers/tty/serial/mfd.c.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14/*
15 * DMA channel allocation:
16 * 1. Even number chans are used for DMA Read (UART TX), odd chans for DMA
17 *    Write (UART RX).
18 * 2. 0/1 channel are assigned to port 0, 2/3 chan to port 1, 4/5 chan to
19 *    port 3, and so on.
20 */
21
22#include <linux/delay.h>
23#include <linux/dmaengine.h>
24#include <linux/dma-mapping.h>
25#include <linux/init.h>
26#include <linux/module.h>
27#include <linux/slab.h>
28
29#include "hsu.h"
30
31#define HSU_DMA_BUSWIDTHS				\
32	BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED)	|	\
33	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE)		|	\
34	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES)		|	\
35	BIT(DMA_SLAVE_BUSWIDTH_3_BYTES)		|	\
36	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)		|	\
37	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)		|	\
38	BIT(DMA_SLAVE_BUSWIDTH_16_BYTES)
39
40static inline void hsu_chan_disable(struct hsu_dma_chan *hsuc)
41{
42	hsu_chan_writel(hsuc, HSU_CH_CR, 0);
43}
44
45static inline void hsu_chan_enable(struct hsu_dma_chan *hsuc)
46{
47	u32 cr = HSU_CH_CR_CHA;
48
49	if (hsuc->direction == DMA_MEM_TO_DEV)
50		cr &= ~HSU_CH_CR_CHD;
51	else if (hsuc->direction == DMA_DEV_TO_MEM)
52		cr |= HSU_CH_CR_CHD;
53
54	hsu_chan_writel(hsuc, HSU_CH_CR, cr);
55}
56
57static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)
58{
59	struct dma_slave_config *config = &hsuc->config;
60	struct hsu_dma_desc *desc = hsuc->desc;
61	u32 bsr = 0, mtsr = 0;	/* to shut the compiler up */
62	u32 dcr = HSU_CH_DCR_CHSOE | HSU_CH_DCR_CHEI;
63	unsigned int i, count;
64
65	if (hsuc->direction == DMA_MEM_TO_DEV) {
66		bsr = config->dst_maxburst;
67		mtsr = config->dst_addr_width;
68	} else if (hsuc->direction == DMA_DEV_TO_MEM) {
69		bsr = config->src_maxburst;
70		mtsr = config->src_addr_width;
71	}
72
73	hsu_chan_disable(hsuc);
74
75	hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
76	hsu_chan_writel(hsuc, HSU_CH_BSR, bsr);
77	hsu_chan_writel(hsuc, HSU_CH_MTSR, mtsr);
78
79	/* Set descriptors */
80	count = (desc->nents - desc->active) % HSU_DMA_CHAN_NR_DESC;
81	for (i = 0; i < count; i++) {
82		hsu_chan_writel(hsuc, HSU_CH_DxSAR(i), desc->sg[i].addr);
83		hsu_chan_writel(hsuc, HSU_CH_DxTSR(i), desc->sg[i].len);
84
85		/* Prepare value for DCR */
86		dcr |= HSU_CH_DCR_DESCA(i);
87		dcr |= HSU_CH_DCR_CHTOI(i);	/* timeout bit, see HSU Errata 1 */
88
89		desc->active++;
90	}
91	/* Only for the last descriptor in the chain */
92	dcr |= HSU_CH_DCR_CHSOD(count - 1);
93	dcr |= HSU_CH_DCR_CHDI(count - 1);
94
95	hsu_chan_writel(hsuc, HSU_CH_DCR, dcr);
96
97	hsu_chan_enable(hsuc);
98}
99
100static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc)
101{
102	unsigned long flags;
103
104	spin_lock_irqsave(&hsuc->lock, flags);
105	hsu_chan_disable(hsuc);
106	hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
107	spin_unlock_irqrestore(&hsuc->lock, flags);
108}
109
110static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc)
111{
112	unsigned long flags;
113
114	spin_lock_irqsave(&hsuc->lock, flags);
115	hsu_dma_chan_start(hsuc);
116	spin_unlock_irqrestore(&hsuc->lock, flags);
117}
118
119static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc)
120{
121	struct virt_dma_desc *vdesc;
122
123	/* Get the next descriptor */
124	vdesc = vchan_next_desc(&hsuc->vchan);
125	if (!vdesc) {
126		hsuc->desc = NULL;
127		return;
128	}
129
130	list_del(&vdesc->node);
131	hsuc->desc = to_hsu_dma_desc(vdesc);
132
133	/* Start the channel with a new descriptor */
134	hsu_dma_start_channel(hsuc);
135}
136
137static u32 hsu_dma_chan_get_sr(struct hsu_dma_chan *hsuc)
138{
139	unsigned long flags;
140	u32 sr;
141
142	spin_lock_irqsave(&hsuc->lock, flags);
143	sr = hsu_chan_readl(hsuc, HSU_CH_SR);
144	spin_unlock_irqrestore(&hsuc->lock, flags);
145
146	return sr & ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY);
147}
148
149irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr)
150{
151	struct hsu_dma_chan *hsuc;
152	struct hsu_dma_desc *desc;
153	unsigned long flags;
154	u32 sr;
155
156	/* Sanity check */
157	if (nr >= chip->pdata->nr_channels)
158		return IRQ_NONE;
159
160	hsuc = &chip->hsu->chan[nr];
161
162	/*
163	 * No matter what situation, need read clear the IRQ status
164	 * There is a bug, see Errata 5, HSD 2900918
165	 */
166	sr = hsu_dma_chan_get_sr(hsuc);
167	if (!sr)
168		return IRQ_NONE;
169
170	/* Timeout IRQ, need wait some time, see Errata 2 */
171	if (hsuc->direction == DMA_DEV_TO_MEM && (sr & HSU_CH_SR_DESCTO_ANY))
172		udelay(2);
173
174	sr &= ~HSU_CH_SR_DESCTO_ANY;
175	if (!sr)
176		return IRQ_HANDLED;
177
178	spin_lock_irqsave(&hsuc->vchan.lock, flags);
179	desc = hsuc->desc;
180	if (desc) {
181		if (sr & HSU_CH_SR_CHE) {
182			desc->status = DMA_ERROR;
183		} else if (desc->active < desc->nents) {
184			hsu_dma_start_channel(hsuc);
185		} else {
186			vchan_cookie_complete(&desc->vdesc);
187			desc->status = DMA_COMPLETE;
188			hsu_dma_start_transfer(hsuc);
189		}
190	}
191	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
192
193	return IRQ_HANDLED;
194}
195EXPORT_SYMBOL_GPL(hsu_dma_irq);
196
197static struct hsu_dma_desc *hsu_dma_alloc_desc(unsigned int nents)
198{
199	struct hsu_dma_desc *desc;
200
201	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
202	if (!desc)
203		return NULL;
204
205	desc->sg = kcalloc(nents, sizeof(*desc->sg), GFP_NOWAIT);
206	if (!desc->sg) {
207		kfree(desc);
208		return NULL;
209	}
210
211	return desc;
212}
213
214static void hsu_dma_desc_free(struct virt_dma_desc *vdesc)
215{
216	struct hsu_dma_desc *desc = to_hsu_dma_desc(vdesc);
217
218	kfree(desc->sg);
219	kfree(desc);
220}
221
222static struct dma_async_tx_descriptor *hsu_dma_prep_slave_sg(
223		struct dma_chan *chan, struct scatterlist *sgl,
224		unsigned int sg_len, enum dma_transfer_direction direction,
225		unsigned long flags, void *context)
226{
227	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
228	struct hsu_dma_desc *desc;
229	struct scatterlist *sg;
230	unsigned int i;
231
232	desc = hsu_dma_alloc_desc(sg_len);
233	if (!desc)
234		return NULL;
235
236	for_each_sg(sgl, sg, sg_len, i) {
237		desc->sg[i].addr = sg_dma_address(sg);
238		desc->sg[i].len = sg_dma_len(sg);
239	}
240
241	desc->nents = sg_len;
242	desc->direction = direction;
243	/* desc->active = 0 by kzalloc */
244	desc->status = DMA_IN_PROGRESS;
245
246	return vchan_tx_prep(&hsuc->vchan, &desc->vdesc, flags);
247}
248
249static void hsu_dma_issue_pending(struct dma_chan *chan)
250{
251	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
252	unsigned long flags;
253
254	spin_lock_irqsave(&hsuc->vchan.lock, flags);
255	if (vchan_issue_pending(&hsuc->vchan) && !hsuc->desc)
256		hsu_dma_start_transfer(hsuc);
257	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
258}
259
260static size_t hsu_dma_desc_size(struct hsu_dma_desc *desc)
261{
262	size_t bytes = 0;
263	unsigned int i;
264
265	for (i = desc->active; i < desc->nents; i++)
266		bytes += desc->sg[i].len;
267
268	return bytes;
269}
270
271static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc)
272{
273	struct hsu_dma_desc *desc = hsuc->desc;
274	size_t bytes = hsu_dma_desc_size(desc);
275	int i;
276	unsigned long flags;
277
278	spin_lock_irqsave(&hsuc->lock, flags);
279	i = desc->active % HSU_DMA_CHAN_NR_DESC;
280	do {
281		bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i));
282	} while (--i >= 0);
283	spin_unlock_irqrestore(&hsuc->lock, flags);
284
285	return bytes;
286}
287
288static enum dma_status hsu_dma_tx_status(struct dma_chan *chan,
289	dma_cookie_t cookie, struct dma_tx_state *state)
290{
291	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
292	struct virt_dma_desc *vdesc;
293	enum dma_status status;
294	size_t bytes;
295	unsigned long flags;
296
297	status = dma_cookie_status(chan, cookie, state);
298	if (status == DMA_COMPLETE)
299		return status;
300
301	spin_lock_irqsave(&hsuc->vchan.lock, flags);
302	vdesc = vchan_find_desc(&hsuc->vchan, cookie);
303	if (hsuc->desc && cookie == hsuc->desc->vdesc.tx.cookie) {
304		bytes = hsu_dma_active_desc_size(hsuc);
305		dma_set_residue(state, bytes);
306		status = hsuc->desc->status;
307	} else if (vdesc) {
308		bytes = hsu_dma_desc_size(to_hsu_dma_desc(vdesc));
309		dma_set_residue(state, bytes);
310	}
311	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
312
313	return status;
314}
315
316static int hsu_dma_slave_config(struct dma_chan *chan,
317				struct dma_slave_config *config)
318{
319	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
320
321	/* Check if chan will be configured for slave transfers */
322	if (!is_slave_direction(config->direction))
323		return -EINVAL;
324
325	memcpy(&hsuc->config, config, sizeof(hsuc->config));
326
327	return 0;
328}
329
330static void hsu_dma_chan_deactivate(struct hsu_dma_chan *hsuc)
331{
332	unsigned long flags;
333
334	spin_lock_irqsave(&hsuc->lock, flags);
335	hsu_chan_disable(hsuc);
336	spin_unlock_irqrestore(&hsuc->lock, flags);
337}
338
339static void hsu_dma_chan_activate(struct hsu_dma_chan *hsuc)
340{
341	unsigned long flags;
342
343	spin_lock_irqsave(&hsuc->lock, flags);
344	hsu_chan_enable(hsuc);
345	spin_unlock_irqrestore(&hsuc->lock, flags);
346}
347
348static int hsu_dma_pause(struct dma_chan *chan)
349{
350	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
351	unsigned long flags;
352
353	spin_lock_irqsave(&hsuc->vchan.lock, flags);
354	if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) {
355		hsu_dma_chan_deactivate(hsuc);
356		hsuc->desc->status = DMA_PAUSED;
357	}
358	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
359
360	return 0;
361}
362
363static int hsu_dma_resume(struct dma_chan *chan)
364{
365	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
366	unsigned long flags;
367
368	spin_lock_irqsave(&hsuc->vchan.lock, flags);
369	if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) {
370		hsuc->desc->status = DMA_IN_PROGRESS;
371		hsu_dma_chan_activate(hsuc);
372	}
373	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
374
375	return 0;
376}
377
378static int hsu_dma_terminate_all(struct dma_chan *chan)
379{
380	struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
381	unsigned long flags;
382	LIST_HEAD(head);
383
384	spin_lock_irqsave(&hsuc->vchan.lock, flags);
385
386	hsu_dma_stop_channel(hsuc);
387	if (hsuc->desc) {
388		hsu_dma_desc_free(&hsuc->desc->vdesc);
389		hsuc->desc = NULL;
390	}
391
392	vchan_get_all_descriptors(&hsuc->vchan, &head);
393	spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
394	vchan_dma_desc_free_list(&hsuc->vchan, &head);
395
396	return 0;
397}
398
399static void hsu_dma_free_chan_resources(struct dma_chan *chan)
400{
401	vchan_free_chan_resources(to_virt_chan(chan));
402}
403
404int hsu_dma_probe(struct hsu_dma_chip *chip)
405{
406	struct hsu_dma *hsu;
407	struct hsu_dma_platform_data *pdata = chip->pdata;
408	void __iomem *addr = chip->regs + chip->offset;
409	unsigned short i;
410	int ret;
411
412	hsu = devm_kzalloc(chip->dev, sizeof(*hsu), GFP_KERNEL);
413	if (!hsu)
414		return -ENOMEM;
415
416	chip->hsu = hsu;
417
418	if (!pdata) {
419		pdata = devm_kzalloc(chip->dev, sizeof(*pdata), GFP_KERNEL);
420		if (!pdata)
421			return -ENOMEM;
422
423		chip->pdata = pdata;
424
425		/* Guess nr_channels from the IO space length */
426		pdata->nr_channels = (chip->length - chip->offset) /
427				     HSU_DMA_CHAN_LENGTH;
428	}
429
430	hsu->chan = devm_kcalloc(chip->dev, pdata->nr_channels,
431				 sizeof(*hsu->chan), GFP_KERNEL);
432	if (!hsu->chan)
433		return -ENOMEM;
434
435	INIT_LIST_HEAD(&hsu->dma.channels);
436	for (i = 0; i < pdata->nr_channels; i++) {
437		struct hsu_dma_chan *hsuc = &hsu->chan[i];
438
439		hsuc->vchan.desc_free = hsu_dma_desc_free;
440		vchan_init(&hsuc->vchan, &hsu->dma);
441
442		hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
443		hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH;
444
445		spin_lock_init(&hsuc->lock);
446	}
447
448	dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask);
449	dma_cap_set(DMA_PRIVATE, hsu->dma.cap_mask);
450
451	hsu->dma.device_free_chan_resources = hsu_dma_free_chan_resources;
452
453	hsu->dma.device_prep_slave_sg = hsu_dma_prep_slave_sg;
454
455	hsu->dma.device_issue_pending = hsu_dma_issue_pending;
456	hsu->dma.device_tx_status = hsu_dma_tx_status;
457
458	hsu->dma.device_config = hsu_dma_slave_config;
459	hsu->dma.device_pause = hsu_dma_pause;
460	hsu->dma.device_resume = hsu_dma_resume;
461	hsu->dma.device_terminate_all = hsu_dma_terminate_all;
462
463	hsu->dma.src_addr_widths = HSU_DMA_BUSWIDTHS;
464	hsu->dma.dst_addr_widths = HSU_DMA_BUSWIDTHS;
465	hsu->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
466	hsu->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
467
468	hsu->dma.dev = chip->dev;
469
470	ret = dma_async_device_register(&hsu->dma);
471	if (ret)
472		return ret;
473
474	dev_info(chip->dev, "Found HSU DMA, %d channels\n", pdata->nr_channels);
475	return 0;
476}
477EXPORT_SYMBOL_GPL(hsu_dma_probe);
478
479int hsu_dma_remove(struct hsu_dma_chip *chip)
480{
481	struct hsu_dma *hsu = chip->hsu;
482	unsigned short i;
483
484	dma_async_device_unregister(&hsu->dma);
485
486	for (i = 0; i < chip->pdata->nr_channels; i++) {
487		struct hsu_dma_chan *hsuc = &hsu->chan[i];
488
489		tasklet_kill(&hsuc->vchan.task);
490	}
491
492	return 0;
493}
494EXPORT_SYMBOL_GPL(hsu_dma_remove);
495
496MODULE_LICENSE("GPL v2");
497MODULE_DESCRIPTION("High Speed UART DMA core driver");
498MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
499