1/* 2 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 3 * http://www.samsung.com 4 * 5 * Copyright (C) 2010 Samsung Electronics Co. Ltd. 6 * Jaswinder Singh <jassi.brar@samsung.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 */ 13 14#include <linux/kernel.h> 15#include <linux/io.h> 16#include <linux/init.h> 17#include <linux/slab.h> 18#include <linux/module.h> 19#include <linux/string.h> 20#include <linux/delay.h> 21#include <linux/interrupt.h> 22#include <linux/dma-mapping.h> 23#include <linux/dmaengine.h> 24#include <linux/amba/bus.h> 25#include <linux/amba/pl330.h> 26#include <linux/scatterlist.h> 27#include <linux/of.h> 28#include <linux/of_dma.h> 29#include <linux/err.h> 30#include <linux/pm_runtime.h> 31 32#include "dmaengine.h" 33#define PL330_MAX_CHAN 8 34#define PL330_MAX_IRQS 32 35#define PL330_MAX_PERI 32 36 37enum pl330_cachectrl { 38 CCTRL0, /* Noncacheable and nonbufferable */ 39 CCTRL1, /* Bufferable only */ 40 CCTRL2, /* Cacheable, but do not allocate */ 41 CCTRL3, /* Cacheable and bufferable, but do not allocate */ 42 INVALID1, /* AWCACHE = 0x1000 */ 43 INVALID2, 44 CCTRL6, /* Cacheable write-through, allocate on writes only */ 45 CCTRL7, /* Cacheable write-back, allocate on writes only */ 46}; 47 48enum pl330_byteswap { 49 SWAP_NO, 50 SWAP_2, 51 SWAP_4, 52 SWAP_8, 53 SWAP_16, 54}; 55 56/* Register and Bit field Definitions */ 57#define DS 0x0 58#define DS_ST_STOP 0x0 59#define DS_ST_EXEC 0x1 60#define DS_ST_CMISS 0x2 61#define DS_ST_UPDTPC 0x3 62#define DS_ST_WFE 0x4 63#define DS_ST_ATBRR 0x5 64#define DS_ST_QBUSY 0x6 65#define DS_ST_WFP 0x7 66#define DS_ST_KILL 0x8 67#define DS_ST_CMPLT 0x9 68#define DS_ST_FLTCMP 0xe 69#define DS_ST_FAULT 0xf 70 71#define DPC 0x4 72#define INTEN 0x20 73#define ES 0x24 74#define INTSTATUS 0x28 75#define INTCLR 0x2c 76#define FSM 0x30 77#define FSC 0x34 78#define FTM 0x38 79 80#define _FTC 0x40 81#define FTC(n) (_FTC + (n)*0x4) 82 83#define _CS 0x100 84#define CS(n) (_CS + (n)*0x8) 85#define CS_CNS (1 << 21) 86 87#define _CPC 0x104 88#define CPC(n) (_CPC + (n)*0x8) 89 90#define _SA 0x400 91#define SA(n) (_SA + (n)*0x20) 92 93#define _DA 0x404 94#define DA(n) (_DA + (n)*0x20) 95 96#define _CC 0x408 97#define CC(n) (_CC + (n)*0x20) 98 99#define CC_SRCINC (1 << 0) 100#define CC_DSTINC (1 << 14) 101#define CC_SRCPRI (1 << 8) 102#define CC_DSTPRI (1 << 22) 103#define CC_SRCNS (1 << 9) 104#define CC_DSTNS (1 << 23) 105#define CC_SRCIA (1 << 10) 106#define CC_DSTIA (1 << 24) 107#define CC_SRCBRSTLEN_SHFT 4 108#define CC_DSTBRSTLEN_SHFT 18 109#define CC_SRCBRSTSIZE_SHFT 1 110#define CC_DSTBRSTSIZE_SHFT 15 111#define CC_SRCCCTRL_SHFT 11 112#define CC_SRCCCTRL_MASK 0x7 113#define CC_DSTCCTRL_SHFT 25 114#define CC_DRCCCTRL_MASK 0x7 115#define CC_SWAP_SHFT 28 116 117#define _LC0 0x40c 118#define LC0(n) (_LC0 + (n)*0x20) 119 120#define _LC1 0x410 121#define LC1(n) (_LC1 + (n)*0x20) 122 123#define DBGSTATUS 0xd00 124#define DBG_BUSY (1 << 0) 125 126#define DBGCMD 0xd04 127#define DBGINST0 0xd08 128#define DBGINST1 0xd0c 129 130#define CR0 0xe00 131#define CR1 0xe04 132#define CR2 0xe08 133#define CR3 0xe0c 134#define CR4 0xe10 135#define CRD 0xe14 136 137#define PERIPH_ID 0xfe0 138#define PERIPH_REV_SHIFT 20 139#define PERIPH_REV_MASK 0xf 140#define PERIPH_REV_R0P0 0 141#define PERIPH_REV_R1P0 1 142#define PERIPH_REV_R1P1 2 143 144#define CR0_PERIPH_REQ_SET (1 << 0) 145#define CR0_BOOT_EN_SET (1 << 1) 146#define CR0_BOOT_MAN_NS (1 << 2) 147#define CR0_NUM_CHANS_SHIFT 4 148#define CR0_NUM_CHANS_MASK 0x7 149#define CR0_NUM_PERIPH_SHIFT 12 150#define CR0_NUM_PERIPH_MASK 0x1f 151#define CR0_NUM_EVENTS_SHIFT 17 152#define CR0_NUM_EVENTS_MASK 0x1f 153 154#define CR1_ICACHE_LEN_SHIFT 0 155#define CR1_ICACHE_LEN_MASK 0x7 156#define CR1_NUM_ICACHELINES_SHIFT 4 157#define CR1_NUM_ICACHELINES_MASK 0xf 158 159#define CRD_DATA_WIDTH_SHIFT 0 160#define CRD_DATA_WIDTH_MASK 0x7 161#define CRD_WR_CAP_SHIFT 4 162#define CRD_WR_CAP_MASK 0x7 163#define CRD_WR_Q_DEP_SHIFT 8 164#define CRD_WR_Q_DEP_MASK 0xf 165#define CRD_RD_CAP_SHIFT 12 166#define CRD_RD_CAP_MASK 0x7 167#define CRD_RD_Q_DEP_SHIFT 16 168#define CRD_RD_Q_DEP_MASK 0xf 169#define CRD_DATA_BUFF_SHIFT 20 170#define CRD_DATA_BUFF_MASK 0x3ff 171 172#define PART 0x330 173#define DESIGNER 0x41 174#define REVISION 0x0 175#define INTEG_CFG 0x0 176#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12)) 177 178#define PL330_STATE_STOPPED (1 << 0) 179#define PL330_STATE_EXECUTING (1 << 1) 180#define PL330_STATE_WFE (1 << 2) 181#define PL330_STATE_FAULTING (1 << 3) 182#define PL330_STATE_COMPLETING (1 << 4) 183#define PL330_STATE_WFP (1 << 5) 184#define PL330_STATE_KILLING (1 << 6) 185#define PL330_STATE_FAULT_COMPLETING (1 << 7) 186#define PL330_STATE_CACHEMISS (1 << 8) 187#define PL330_STATE_UPDTPC (1 << 9) 188#define PL330_STATE_ATBARRIER (1 << 10) 189#define PL330_STATE_QUEUEBUSY (1 << 11) 190#define PL330_STATE_INVALID (1 << 15) 191 192#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \ 193 | PL330_STATE_WFE | PL330_STATE_FAULTING) 194 195#define CMD_DMAADDH 0x54 196#define CMD_DMAEND 0x00 197#define CMD_DMAFLUSHP 0x35 198#define CMD_DMAGO 0xa0 199#define CMD_DMALD 0x04 200#define CMD_DMALDP 0x25 201#define CMD_DMALP 0x20 202#define CMD_DMALPEND 0x28 203#define CMD_DMAKILL 0x01 204#define CMD_DMAMOV 0xbc 205#define CMD_DMANOP 0x18 206#define CMD_DMARMB 0x12 207#define CMD_DMASEV 0x34 208#define CMD_DMAST 0x08 209#define CMD_DMASTP 0x29 210#define CMD_DMASTZ 0x0c 211#define CMD_DMAWFE 0x36 212#define CMD_DMAWFP 0x30 213#define CMD_DMAWMB 0x13 214 215#define SZ_DMAADDH 3 216#define SZ_DMAEND 1 217#define SZ_DMAFLUSHP 2 218#define SZ_DMALD 1 219#define SZ_DMALDP 2 220#define SZ_DMALP 2 221#define SZ_DMALPEND 2 222#define SZ_DMAKILL 1 223#define SZ_DMAMOV 6 224#define SZ_DMANOP 1 225#define SZ_DMARMB 1 226#define SZ_DMASEV 2 227#define SZ_DMAST 1 228#define SZ_DMASTP 2 229#define SZ_DMASTZ 1 230#define SZ_DMAWFE 2 231#define SZ_DMAWFP 2 232#define SZ_DMAWMB 1 233#define SZ_DMAGO 6 234 235#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1) 236#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7)) 237 238#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr)) 239#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr)) 240 241/* 242 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req 243 * at 1byte/burst for P<->M and M<->M respectively. 244 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req 245 * should be enough for P<->M and M<->M respectively. 246 */ 247#define MCODE_BUFF_PER_REQ 256 248 249/* Use this _only_ to wait on transient states */ 250#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax(); 251 252#ifdef PL330_DEBUG_MCGEN 253static unsigned cmd_line; 254#define PL330_DBGCMD_DUMP(off, x...) do { \ 255 printk("%x:", cmd_line); \ 256 printk(x); \ 257 cmd_line += off; \ 258 } while (0) 259#define PL330_DBGMC_START(addr) (cmd_line = addr) 260#else 261#define PL330_DBGCMD_DUMP(off, x...) do {} while (0) 262#define PL330_DBGMC_START(addr) do {} while (0) 263#endif 264 265/* The number of default descriptors */ 266 267#define NR_DEFAULT_DESC 16 268 269/* Delay for runtime PM autosuspend, ms */ 270#define PL330_AUTOSUSPEND_DELAY 20 271 272/* Populated by the PL330 core driver for DMA API driver's info */ 273struct pl330_config { 274 u32 periph_id; 275#define DMAC_MODE_NS (1 << 0) 276 unsigned int mode; 277 unsigned int data_bus_width:10; /* In number of bits */ 278 unsigned int data_buf_dep:11; 279 unsigned int num_chan:4; 280 unsigned int num_peri:6; 281 u32 peri_ns; 282 unsigned int num_events:6; 283 u32 irq_ns; 284}; 285 286/** 287 * Request Configuration. 288 * The PL330 core does not modify this and uses the last 289 * working configuration if the request doesn't provide any. 290 * 291 * The Client may want to provide this info only for the 292 * first request and a request with new settings. 293 */ 294struct pl330_reqcfg { 295 /* Address Incrementing */ 296 unsigned dst_inc:1; 297 unsigned src_inc:1; 298 299 /* 300 * For now, the SRC & DST protection levels 301 * and burst size/length are assumed same. 302 */ 303 bool nonsecure; 304 bool privileged; 305 bool insnaccess; 306 unsigned brst_len:5; 307 unsigned brst_size:3; /* in power of 2 */ 308 309 enum pl330_cachectrl dcctl; 310 enum pl330_cachectrl scctl; 311 enum pl330_byteswap swap; 312 struct pl330_config *pcfg; 313}; 314 315/* 316 * One cycle of DMAC operation. 317 * There may be more than one xfer in a request. 318 */ 319struct pl330_xfer { 320 u32 src_addr; 321 u32 dst_addr; 322 /* Size to xfer */ 323 u32 bytes; 324}; 325 326/* The xfer callbacks are made with one of these arguments. */ 327enum pl330_op_err { 328 /* The all xfers in the request were success. */ 329 PL330_ERR_NONE, 330 /* If req aborted due to global error. */ 331 PL330_ERR_ABORT, 332 /* If req failed due to problem with Channel. */ 333 PL330_ERR_FAIL, 334}; 335 336enum dmamov_dst { 337 SAR = 0, 338 CCR, 339 DAR, 340}; 341 342enum pl330_dst { 343 SRC = 0, 344 DST, 345}; 346 347enum pl330_cond { 348 SINGLE, 349 BURST, 350 ALWAYS, 351}; 352 353struct dma_pl330_desc; 354 355struct _pl330_req { 356 u32 mc_bus; 357 void *mc_cpu; 358 struct dma_pl330_desc *desc; 359}; 360 361/* ToBeDone for tasklet */ 362struct _pl330_tbd { 363 bool reset_dmac; 364 bool reset_mngr; 365 u8 reset_chan; 366}; 367 368/* A DMAC Thread */ 369struct pl330_thread { 370 u8 id; 371 int ev; 372 /* If the channel is not yet acquired by any client */ 373 bool free; 374 /* Parent DMAC */ 375 struct pl330_dmac *dmac; 376 /* Only two at a time */ 377 struct _pl330_req req[2]; 378 /* Index of the last enqueued request */ 379 unsigned lstenq; 380 /* Index of the last submitted request or -1 if the DMA is stopped */ 381 int req_running; 382}; 383 384enum pl330_dmac_state { 385 UNINIT, 386 INIT, 387 DYING, 388}; 389 390enum desc_status { 391 /* In the DMAC pool */ 392 FREE, 393 /* 394 * Allocated to some channel during prep_xxx 395 * Also may be sitting on the work_list. 396 */ 397 PREP, 398 /* 399 * Sitting on the work_list and already submitted 400 * to the PL330 core. Not more than two descriptors 401 * of a channel can be BUSY at any time. 402 */ 403 BUSY, 404 /* 405 * Sitting on the channel work_list but xfer done 406 * by PL330 core 407 */ 408 DONE, 409}; 410 411struct dma_pl330_chan { 412 /* Schedule desc completion */ 413 struct tasklet_struct task; 414 415 /* DMA-Engine Channel */ 416 struct dma_chan chan; 417 418 /* List of submitted descriptors */ 419 struct list_head submitted_list; 420 /* List of issued descriptors */ 421 struct list_head work_list; 422 /* List of completed descriptors */ 423 struct list_head completed_list; 424 425 /* Pointer to the DMAC that manages this channel, 426 * NULL if the channel is available to be acquired. 427 * As the parent, this DMAC also provides descriptors 428 * to the channel. 429 */ 430 struct pl330_dmac *dmac; 431 432 /* To protect channel manipulation */ 433 spinlock_t lock; 434 435 /* 436 * Hardware channel thread of PL330 DMAC. NULL if the channel is 437 * available. 438 */ 439 struct pl330_thread *thread; 440 441 /* For D-to-M and M-to-D channels */ 442 int burst_sz; /* the peripheral fifo width */ 443 int burst_len; /* the number of burst */ 444 dma_addr_t fifo_addr; 445 446 /* for cyclic capability */ 447 bool cyclic; 448}; 449 450struct pl330_dmac { 451 /* DMA-Engine Device */ 452 struct dma_device ddma; 453 454 /* Holds info about sg limitations */ 455 struct device_dma_parameters dma_parms; 456 457 /* Pool of descriptors available for the DMAC's channels */ 458 struct list_head desc_pool; 459 /* To protect desc_pool manipulation */ 460 spinlock_t pool_lock; 461 462 /* Size of MicroCode buffers for each channel. */ 463 unsigned mcbufsz; 464 /* ioremap'ed address of PL330 registers. */ 465 void __iomem *base; 466 /* Populated by the PL330 core driver during pl330_add */ 467 struct pl330_config pcfg; 468 469 spinlock_t lock; 470 /* Maximum possible events/irqs */ 471 int events[32]; 472 /* BUS address of MicroCode buffer */ 473 dma_addr_t mcode_bus; 474 /* CPU address of MicroCode buffer */ 475 void *mcode_cpu; 476 /* List of all Channel threads */ 477 struct pl330_thread *channels; 478 /* Pointer to the MANAGER thread */ 479 struct pl330_thread *manager; 480 /* To handle bad news in interrupt */ 481 struct tasklet_struct tasks; 482 struct _pl330_tbd dmac_tbd; 483 /* State of DMAC operation */ 484 enum pl330_dmac_state state; 485 /* Holds list of reqs with due callbacks */ 486 struct list_head req_done; 487 488 /* Peripheral channels connected to this DMAC */ 489 unsigned int num_peripherals; 490 struct dma_pl330_chan *peripherals; /* keep at end */ 491}; 492 493struct dma_pl330_desc { 494 /* To attach to a queue as child */ 495 struct list_head node; 496 497 /* Descriptor for the DMA Engine API */ 498 struct dma_async_tx_descriptor txd; 499 500 /* Xfer for PL330 core */ 501 struct pl330_xfer px; 502 503 struct pl330_reqcfg rqcfg; 504 505 enum desc_status status; 506 507 int bytes_requested; 508 bool last; 509 510 /* The channel which currently holds this desc */ 511 struct dma_pl330_chan *pchan; 512 513 enum dma_transfer_direction rqtype; 514 /* Index of peripheral for the xfer. */ 515 unsigned peri:5; 516 /* Hook to attach to DMAC's list of reqs with due callback */ 517 struct list_head rqd; 518}; 519 520struct _xfer_spec { 521 u32 ccr; 522 struct dma_pl330_desc *desc; 523}; 524 525static inline bool _queue_empty(struct pl330_thread *thrd) 526{ 527 return thrd->req[0].desc == NULL && thrd->req[1].desc == NULL; 528} 529 530static inline bool _queue_full(struct pl330_thread *thrd) 531{ 532 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL; 533} 534 535static inline bool is_manager(struct pl330_thread *thrd) 536{ 537 return thrd->dmac->manager == thrd; 538} 539 540/* If manager of the thread is in Non-Secure mode */ 541static inline bool _manager_ns(struct pl330_thread *thrd) 542{ 543 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false; 544} 545 546static inline u32 get_revision(u32 periph_id) 547{ 548 return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK; 549} 550 551static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[], 552 enum pl330_dst da, u16 val) 553{ 554 if (dry_run) 555 return SZ_DMAADDH; 556 557 buf[0] = CMD_DMAADDH; 558 buf[0] |= (da << 1); 559 *((__le16 *)&buf[1]) = cpu_to_le16(val); 560 561 PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n", 562 da == 1 ? "DA" : "SA", val); 563 564 return SZ_DMAADDH; 565} 566 567static inline u32 _emit_END(unsigned dry_run, u8 buf[]) 568{ 569 if (dry_run) 570 return SZ_DMAEND; 571 572 buf[0] = CMD_DMAEND; 573 574 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n"); 575 576 return SZ_DMAEND; 577} 578 579static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri) 580{ 581 if (dry_run) 582 return SZ_DMAFLUSHP; 583 584 buf[0] = CMD_DMAFLUSHP; 585 586 peri &= 0x1f; 587 peri <<= 3; 588 buf[1] = peri; 589 590 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3); 591 592 return SZ_DMAFLUSHP; 593} 594 595static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond) 596{ 597 if (dry_run) 598 return SZ_DMALD; 599 600 buf[0] = CMD_DMALD; 601 602 if (cond == SINGLE) 603 buf[0] |= (0 << 1) | (1 << 0); 604 else if (cond == BURST) 605 buf[0] |= (1 << 1) | (1 << 0); 606 607 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n", 608 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); 609 610 return SZ_DMALD; 611} 612 613static inline u32 _emit_LDP(unsigned dry_run, u8 buf[], 614 enum pl330_cond cond, u8 peri) 615{ 616 if (dry_run) 617 return SZ_DMALDP; 618 619 buf[0] = CMD_DMALDP; 620 621 if (cond == BURST) 622 buf[0] |= (1 << 1); 623 624 peri &= 0x1f; 625 peri <<= 3; 626 buf[1] = peri; 627 628 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n", 629 cond == SINGLE ? 'S' : 'B', peri >> 3); 630 631 return SZ_DMALDP; 632} 633 634static inline u32 _emit_LP(unsigned dry_run, u8 buf[], 635 unsigned loop, u8 cnt) 636{ 637 if (dry_run) 638 return SZ_DMALP; 639 640 buf[0] = CMD_DMALP; 641 642 if (loop) 643 buf[0] |= (1 << 1); 644 645 cnt--; /* DMAC increments by 1 internally */ 646 buf[1] = cnt; 647 648 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt); 649 650 return SZ_DMALP; 651} 652 653struct _arg_LPEND { 654 enum pl330_cond cond; 655 bool forever; 656 unsigned loop; 657 u8 bjump; 658}; 659 660static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[], 661 const struct _arg_LPEND *arg) 662{ 663 enum pl330_cond cond = arg->cond; 664 bool forever = arg->forever; 665 unsigned loop = arg->loop; 666 u8 bjump = arg->bjump; 667 668 if (dry_run) 669 return SZ_DMALPEND; 670 671 buf[0] = CMD_DMALPEND; 672 673 if (loop) 674 buf[0] |= (1 << 2); 675 676 if (!forever) 677 buf[0] |= (1 << 4); 678 679 if (cond == SINGLE) 680 buf[0] |= (0 << 1) | (1 << 0); 681 else if (cond == BURST) 682 buf[0] |= (1 << 1) | (1 << 0); 683 684 buf[1] = bjump; 685 686 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n", 687 forever ? "FE" : "END", 688 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'), 689 loop ? '1' : '0', 690 bjump); 691 692 return SZ_DMALPEND; 693} 694 695static inline u32 _emit_KILL(unsigned dry_run, u8 buf[]) 696{ 697 if (dry_run) 698 return SZ_DMAKILL; 699 700 buf[0] = CMD_DMAKILL; 701 702 return SZ_DMAKILL; 703} 704 705static inline u32 _emit_MOV(unsigned dry_run, u8 buf[], 706 enum dmamov_dst dst, u32 val) 707{ 708 if (dry_run) 709 return SZ_DMAMOV; 710 711 buf[0] = CMD_DMAMOV; 712 buf[1] = dst; 713 *((__le32 *)&buf[2]) = cpu_to_le32(val); 714 715 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n", 716 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val); 717 718 return SZ_DMAMOV; 719} 720 721static inline u32 _emit_NOP(unsigned dry_run, u8 buf[]) 722{ 723 if (dry_run) 724 return SZ_DMANOP; 725 726 buf[0] = CMD_DMANOP; 727 728 PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n"); 729 730 return SZ_DMANOP; 731} 732 733static inline u32 _emit_RMB(unsigned dry_run, u8 buf[]) 734{ 735 if (dry_run) 736 return SZ_DMARMB; 737 738 buf[0] = CMD_DMARMB; 739 740 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n"); 741 742 return SZ_DMARMB; 743} 744 745static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev) 746{ 747 if (dry_run) 748 return SZ_DMASEV; 749 750 buf[0] = CMD_DMASEV; 751 752 ev &= 0x1f; 753 ev <<= 3; 754 buf[1] = ev; 755 756 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3); 757 758 return SZ_DMASEV; 759} 760 761static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond) 762{ 763 if (dry_run) 764 return SZ_DMAST; 765 766 buf[0] = CMD_DMAST; 767 768 if (cond == SINGLE) 769 buf[0] |= (0 << 1) | (1 << 0); 770 else if (cond == BURST) 771 buf[0] |= (1 << 1) | (1 << 0); 772 773 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n", 774 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); 775 776 return SZ_DMAST; 777} 778 779static inline u32 _emit_STP(unsigned dry_run, u8 buf[], 780 enum pl330_cond cond, u8 peri) 781{ 782 if (dry_run) 783 return SZ_DMASTP; 784 785 buf[0] = CMD_DMASTP; 786 787 if (cond == BURST) 788 buf[0] |= (1 << 1); 789 790 peri &= 0x1f; 791 peri <<= 3; 792 buf[1] = peri; 793 794 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n", 795 cond == SINGLE ? 'S' : 'B', peri >> 3); 796 797 return SZ_DMASTP; 798} 799 800static inline u32 _emit_STZ(unsigned dry_run, u8 buf[]) 801{ 802 if (dry_run) 803 return SZ_DMASTZ; 804 805 buf[0] = CMD_DMASTZ; 806 807 PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n"); 808 809 return SZ_DMASTZ; 810} 811 812static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev, 813 unsigned invalidate) 814{ 815 if (dry_run) 816 return SZ_DMAWFE; 817 818 buf[0] = CMD_DMAWFE; 819 820 ev &= 0x1f; 821 ev <<= 3; 822 buf[1] = ev; 823 824 if (invalidate) 825 buf[1] |= (1 << 1); 826 827 PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n", 828 ev >> 3, invalidate ? ", I" : ""); 829 830 return SZ_DMAWFE; 831} 832 833static inline u32 _emit_WFP(unsigned dry_run, u8 buf[], 834 enum pl330_cond cond, u8 peri) 835{ 836 if (dry_run) 837 return SZ_DMAWFP; 838 839 buf[0] = CMD_DMAWFP; 840 841 if (cond == SINGLE) 842 buf[0] |= (0 << 1) | (0 << 0); 843 else if (cond == BURST) 844 buf[0] |= (1 << 1) | (0 << 0); 845 else 846 buf[0] |= (0 << 1) | (1 << 0); 847 848 peri &= 0x1f; 849 peri <<= 3; 850 buf[1] = peri; 851 852 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n", 853 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3); 854 855 return SZ_DMAWFP; 856} 857 858static inline u32 _emit_WMB(unsigned dry_run, u8 buf[]) 859{ 860 if (dry_run) 861 return SZ_DMAWMB; 862 863 buf[0] = CMD_DMAWMB; 864 865 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n"); 866 867 return SZ_DMAWMB; 868} 869 870struct _arg_GO { 871 u8 chan; 872 u32 addr; 873 unsigned ns; 874}; 875 876static inline u32 _emit_GO(unsigned dry_run, u8 buf[], 877 const struct _arg_GO *arg) 878{ 879 u8 chan = arg->chan; 880 u32 addr = arg->addr; 881 unsigned ns = arg->ns; 882 883 if (dry_run) 884 return SZ_DMAGO; 885 886 buf[0] = CMD_DMAGO; 887 buf[0] |= (ns << 1); 888 889 buf[1] = chan & 0x7; 890 891 *((__le32 *)&buf[2]) = cpu_to_le32(addr); 892 893 return SZ_DMAGO; 894} 895 896#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) 897 898/* Returns Time-Out */ 899static bool _until_dmac_idle(struct pl330_thread *thrd) 900{ 901 void __iomem *regs = thrd->dmac->base; 902 unsigned long loops = msecs_to_loops(5); 903 904 do { 905 /* Until Manager is Idle */ 906 if (!(readl(regs + DBGSTATUS) & DBG_BUSY)) 907 break; 908 909 cpu_relax(); 910 } while (--loops); 911 912 if (!loops) 913 return true; 914 915 return false; 916} 917 918static inline void _execute_DBGINSN(struct pl330_thread *thrd, 919 u8 insn[], bool as_manager) 920{ 921 void __iomem *regs = thrd->dmac->base; 922 u32 val; 923 924 val = (insn[0] << 16) | (insn[1] << 24); 925 if (!as_manager) { 926 val |= (1 << 0); 927 val |= (thrd->id << 8); /* Channel Number */ 928 } 929 writel(val, regs + DBGINST0); 930 931 val = le32_to_cpu(*((__le32 *)&insn[2])); 932 writel(val, regs + DBGINST1); 933 934 /* If timed out due to halted state-machine */ 935 if (_until_dmac_idle(thrd)) { 936 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n"); 937 return; 938 } 939 940 /* Get going */ 941 writel(0, regs + DBGCMD); 942} 943 944static inline u32 _state(struct pl330_thread *thrd) 945{ 946 void __iomem *regs = thrd->dmac->base; 947 u32 val; 948 949 if (is_manager(thrd)) 950 val = readl(regs + DS) & 0xf; 951 else 952 val = readl(regs + CS(thrd->id)) & 0xf; 953 954 switch (val) { 955 case DS_ST_STOP: 956 return PL330_STATE_STOPPED; 957 case DS_ST_EXEC: 958 return PL330_STATE_EXECUTING; 959 case DS_ST_CMISS: 960 return PL330_STATE_CACHEMISS; 961 case DS_ST_UPDTPC: 962 return PL330_STATE_UPDTPC; 963 case DS_ST_WFE: 964 return PL330_STATE_WFE; 965 case DS_ST_FAULT: 966 return PL330_STATE_FAULTING; 967 case DS_ST_ATBRR: 968 if (is_manager(thrd)) 969 return PL330_STATE_INVALID; 970 else 971 return PL330_STATE_ATBARRIER; 972 case DS_ST_QBUSY: 973 if (is_manager(thrd)) 974 return PL330_STATE_INVALID; 975 else 976 return PL330_STATE_QUEUEBUSY; 977 case DS_ST_WFP: 978 if (is_manager(thrd)) 979 return PL330_STATE_INVALID; 980 else 981 return PL330_STATE_WFP; 982 case DS_ST_KILL: 983 if (is_manager(thrd)) 984 return PL330_STATE_INVALID; 985 else 986 return PL330_STATE_KILLING; 987 case DS_ST_CMPLT: 988 if (is_manager(thrd)) 989 return PL330_STATE_INVALID; 990 else 991 return PL330_STATE_COMPLETING; 992 case DS_ST_FLTCMP: 993 if (is_manager(thrd)) 994 return PL330_STATE_INVALID; 995 else 996 return PL330_STATE_FAULT_COMPLETING; 997 default: 998 return PL330_STATE_INVALID; 999 } 1000} 1001 1002static void _stop(struct pl330_thread *thrd) 1003{ 1004 void __iomem *regs = thrd->dmac->base; 1005 u8 insn[6] = {0, 0, 0, 0, 0, 0}; 1006 1007 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING) 1008 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING); 1009 1010 /* Return if nothing needs to be done */ 1011 if (_state(thrd) == PL330_STATE_COMPLETING 1012 || _state(thrd) == PL330_STATE_KILLING 1013 || _state(thrd) == PL330_STATE_STOPPED) 1014 return; 1015 1016 _emit_KILL(0, insn); 1017 1018 /* Stop generating interrupts for SEV */ 1019 writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN); 1020 1021 _execute_DBGINSN(thrd, insn, is_manager(thrd)); 1022} 1023 1024/* Start doing req 'idx' of thread 'thrd' */ 1025static bool _trigger(struct pl330_thread *thrd) 1026{ 1027 void __iomem *regs = thrd->dmac->base; 1028 struct _pl330_req *req; 1029 struct dma_pl330_desc *desc; 1030 struct _arg_GO go; 1031 unsigned ns; 1032 u8 insn[6] = {0, 0, 0, 0, 0, 0}; 1033 int idx; 1034 1035 /* Return if already ACTIVE */ 1036 if (_state(thrd) != PL330_STATE_STOPPED) 1037 return true; 1038 1039 idx = 1 - thrd->lstenq; 1040 if (thrd->req[idx].desc != NULL) { 1041 req = &thrd->req[idx]; 1042 } else { 1043 idx = thrd->lstenq; 1044 if (thrd->req[idx].desc != NULL) 1045 req = &thrd->req[idx]; 1046 else 1047 req = NULL; 1048 } 1049 1050 /* Return if no request */ 1051 if (!req) 1052 return true; 1053 1054 /* Return if req is running */ 1055 if (idx == thrd->req_running) 1056 return true; 1057 1058 desc = req->desc; 1059 1060 ns = desc->rqcfg.nonsecure ? 1 : 0; 1061 1062 /* See 'Abort Sources' point-4 at Page 2-25 */ 1063 if (_manager_ns(thrd) && !ns) 1064 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n", 1065 __func__, __LINE__); 1066 1067 go.chan = thrd->id; 1068 go.addr = req->mc_bus; 1069 go.ns = ns; 1070 _emit_GO(0, insn, &go); 1071 1072 /* Set to generate interrupts for SEV */ 1073 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN); 1074 1075 /* Only manager can execute GO */ 1076 _execute_DBGINSN(thrd, insn, true); 1077 1078 thrd->req_running = idx; 1079 1080 return true; 1081} 1082 1083static bool _start(struct pl330_thread *thrd) 1084{ 1085 switch (_state(thrd)) { 1086 case PL330_STATE_FAULT_COMPLETING: 1087 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING); 1088 1089 if (_state(thrd) == PL330_STATE_KILLING) 1090 UNTIL(thrd, PL330_STATE_STOPPED) 1091 1092 case PL330_STATE_FAULTING: 1093 _stop(thrd); 1094 1095 case PL330_STATE_KILLING: 1096 case PL330_STATE_COMPLETING: 1097 UNTIL(thrd, PL330_STATE_STOPPED) 1098 1099 case PL330_STATE_STOPPED: 1100 return _trigger(thrd); 1101 1102 case PL330_STATE_WFP: 1103 case PL330_STATE_QUEUEBUSY: 1104 case PL330_STATE_ATBARRIER: 1105 case PL330_STATE_UPDTPC: 1106 case PL330_STATE_CACHEMISS: 1107 case PL330_STATE_EXECUTING: 1108 return true; 1109 1110 case PL330_STATE_WFE: /* For RESUME, nothing yet */ 1111 default: 1112 return false; 1113 } 1114} 1115 1116static inline int _ldst_memtomem(unsigned dry_run, u8 buf[], 1117 const struct _xfer_spec *pxs, int cyc) 1118{ 1119 int off = 0; 1120 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg; 1121 1122 /* check lock-up free version */ 1123 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) { 1124 while (cyc--) { 1125 off += _emit_LD(dry_run, &buf[off], ALWAYS); 1126 off += _emit_ST(dry_run, &buf[off], ALWAYS); 1127 } 1128 } else { 1129 while (cyc--) { 1130 off += _emit_LD(dry_run, &buf[off], ALWAYS); 1131 off += _emit_RMB(dry_run, &buf[off]); 1132 off += _emit_ST(dry_run, &buf[off], ALWAYS); 1133 off += _emit_WMB(dry_run, &buf[off]); 1134 } 1135 } 1136 1137 return off; 1138} 1139 1140static inline int _ldst_devtomem(unsigned dry_run, u8 buf[], 1141 const struct _xfer_spec *pxs, int cyc) 1142{ 1143 int off = 0; 1144 1145 while (cyc--) { 1146 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri); 1147 off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->desc->peri); 1148 off += _emit_ST(dry_run, &buf[off], ALWAYS); 1149 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); 1150 } 1151 1152 return off; 1153} 1154 1155static inline int _ldst_memtodev(unsigned dry_run, u8 buf[], 1156 const struct _xfer_spec *pxs, int cyc) 1157{ 1158 int off = 0; 1159 1160 while (cyc--) { 1161 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri); 1162 off += _emit_LD(dry_run, &buf[off], ALWAYS); 1163 off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->desc->peri); 1164 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri); 1165 } 1166 1167 return off; 1168} 1169 1170static int _bursts(unsigned dry_run, u8 buf[], 1171 const struct _xfer_spec *pxs, int cyc) 1172{ 1173 int off = 0; 1174 1175 switch (pxs->desc->rqtype) { 1176 case DMA_MEM_TO_DEV: 1177 off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc); 1178 break; 1179 case DMA_DEV_TO_MEM: 1180 off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc); 1181 break; 1182 case DMA_MEM_TO_MEM: 1183 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc); 1184 break; 1185 default: 1186 off += 0x40000000; /* Scare off the Client */ 1187 break; 1188 } 1189 1190 return off; 1191} 1192 1193/* Returns bytes consumed and updates bursts */ 1194static inline int _loop(unsigned dry_run, u8 buf[], 1195 unsigned long *bursts, const struct _xfer_spec *pxs) 1196{ 1197 int cyc, cycmax, szlp, szlpend, szbrst, off; 1198 unsigned lcnt0, lcnt1, ljmp0, ljmp1; 1199 struct _arg_LPEND lpend; 1200 1201 /* Max iterations possible in DMALP is 256 */ 1202 if (*bursts >= 256*256) { 1203 lcnt1 = 256; 1204 lcnt0 = 256; 1205 cyc = *bursts / lcnt1 / lcnt0; 1206 } else if (*bursts > 256) { 1207 lcnt1 = 256; 1208 lcnt0 = *bursts / lcnt1; 1209 cyc = 1; 1210 } else { 1211 lcnt1 = *bursts; 1212 lcnt0 = 0; 1213 cyc = 1; 1214 } 1215 1216 szlp = _emit_LP(1, buf, 0, 0); 1217 szbrst = _bursts(1, buf, pxs, 1); 1218 1219 lpend.cond = ALWAYS; 1220 lpend.forever = false; 1221 lpend.loop = 0; 1222 lpend.bjump = 0; 1223 szlpend = _emit_LPEND(1, buf, &lpend); 1224 1225 if (lcnt0) { 1226 szlp *= 2; 1227 szlpend *= 2; 1228 } 1229 1230 /* 1231 * Max bursts that we can unroll due to limit on the 1232 * size of backward jump that can be encoded in DMALPEND 1233 * which is 8-bits and hence 255 1234 */ 1235 cycmax = (255 - (szlp + szlpend)) / szbrst; 1236 1237 cyc = (cycmax < cyc) ? cycmax : cyc; 1238 1239 off = 0; 1240 1241 if (lcnt0) { 1242 off += _emit_LP(dry_run, &buf[off], 0, lcnt0); 1243 ljmp0 = off; 1244 } 1245 1246 off += _emit_LP(dry_run, &buf[off], 1, lcnt1); 1247 ljmp1 = off; 1248 1249 off += _bursts(dry_run, &buf[off], pxs, cyc); 1250 1251 lpend.cond = ALWAYS; 1252 lpend.forever = false; 1253 lpend.loop = 1; 1254 lpend.bjump = off - ljmp1; 1255 off += _emit_LPEND(dry_run, &buf[off], &lpend); 1256 1257 if (lcnt0) { 1258 lpend.cond = ALWAYS; 1259 lpend.forever = false; 1260 lpend.loop = 0; 1261 lpend.bjump = off - ljmp0; 1262 off += _emit_LPEND(dry_run, &buf[off], &lpend); 1263 } 1264 1265 *bursts = lcnt1 * cyc; 1266 if (lcnt0) 1267 *bursts *= lcnt0; 1268 1269 return off; 1270} 1271 1272static inline int _setup_loops(unsigned dry_run, u8 buf[], 1273 const struct _xfer_spec *pxs) 1274{ 1275 struct pl330_xfer *x = &pxs->desc->px; 1276 u32 ccr = pxs->ccr; 1277 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr); 1278 int off = 0; 1279 1280 while (bursts) { 1281 c = bursts; 1282 off += _loop(dry_run, &buf[off], &c, pxs); 1283 bursts -= c; 1284 } 1285 1286 return off; 1287} 1288 1289static inline int _setup_xfer(unsigned dry_run, u8 buf[], 1290 const struct _xfer_spec *pxs) 1291{ 1292 struct pl330_xfer *x = &pxs->desc->px; 1293 int off = 0; 1294 1295 /* DMAMOV SAR, x->src_addr */ 1296 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr); 1297 /* DMAMOV DAR, x->dst_addr */ 1298 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr); 1299 1300 /* Setup Loop(s) */ 1301 off += _setup_loops(dry_run, &buf[off], pxs); 1302 1303 return off; 1304} 1305 1306/* 1307 * A req is a sequence of one or more xfer units. 1308 * Returns the number of bytes taken to setup the MC for the req. 1309 */ 1310static int _setup_req(unsigned dry_run, struct pl330_thread *thrd, 1311 unsigned index, struct _xfer_spec *pxs) 1312{ 1313 struct _pl330_req *req = &thrd->req[index]; 1314 struct pl330_xfer *x; 1315 u8 *buf = req->mc_cpu; 1316 int off = 0; 1317 1318 PL330_DBGMC_START(req->mc_bus); 1319 1320 /* DMAMOV CCR, ccr */ 1321 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr); 1322 1323 x = &pxs->desc->px; 1324 /* Error if xfer length is not aligned at burst size */ 1325 if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr))) 1326 return -EINVAL; 1327 1328 off += _setup_xfer(dry_run, &buf[off], pxs); 1329 1330 /* DMASEV peripheral/event */ 1331 off += _emit_SEV(dry_run, &buf[off], thrd->ev); 1332 /* DMAEND */ 1333 off += _emit_END(dry_run, &buf[off]); 1334 1335 return off; 1336} 1337 1338static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc) 1339{ 1340 u32 ccr = 0; 1341 1342 if (rqc->src_inc) 1343 ccr |= CC_SRCINC; 1344 1345 if (rqc->dst_inc) 1346 ccr |= CC_DSTINC; 1347 1348 /* We set same protection levels for Src and DST for now */ 1349 if (rqc->privileged) 1350 ccr |= CC_SRCPRI | CC_DSTPRI; 1351 if (rqc->nonsecure) 1352 ccr |= CC_SRCNS | CC_DSTNS; 1353 if (rqc->insnaccess) 1354 ccr |= CC_SRCIA | CC_DSTIA; 1355 1356 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT); 1357 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT); 1358 1359 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT); 1360 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT); 1361 1362 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT); 1363 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT); 1364 1365 ccr |= (rqc->swap << CC_SWAP_SHFT); 1366 1367 return ccr; 1368} 1369 1370/* 1371 * Submit a list of xfers after which the client wants notification. 1372 * Client is not notified after each xfer unit, just once after all 1373 * xfer units are done or some error occurs. 1374 */ 1375static int pl330_submit_req(struct pl330_thread *thrd, 1376 struct dma_pl330_desc *desc) 1377{ 1378 struct pl330_dmac *pl330 = thrd->dmac; 1379 struct _xfer_spec xs; 1380 unsigned long flags; 1381 unsigned idx; 1382 u32 ccr; 1383 int ret = 0; 1384 1385 if (pl330->state == DYING 1386 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) { 1387 dev_info(thrd->dmac->ddma.dev, "%s:%d\n", 1388 __func__, __LINE__); 1389 return -EAGAIN; 1390 } 1391 1392 /* If request for non-existing peripheral */ 1393 if (desc->rqtype != DMA_MEM_TO_MEM && 1394 desc->peri >= pl330->pcfg.num_peri) { 1395 dev_info(thrd->dmac->ddma.dev, 1396 "%s:%d Invalid peripheral(%u)!\n", 1397 __func__, __LINE__, desc->peri); 1398 return -EINVAL; 1399 } 1400 1401 spin_lock_irqsave(&pl330->lock, flags); 1402 1403 if (_queue_full(thrd)) { 1404 ret = -EAGAIN; 1405 goto xfer_exit; 1406 } 1407 1408 /* Prefer Secure Channel */ 1409 if (!_manager_ns(thrd)) 1410 desc->rqcfg.nonsecure = 0; 1411 else 1412 desc->rqcfg.nonsecure = 1; 1413 1414 ccr = _prepare_ccr(&desc->rqcfg); 1415 1416 idx = thrd->req[0].desc == NULL ? 0 : 1; 1417 1418 xs.ccr = ccr; 1419 xs.desc = desc; 1420 1421 /* First dry run to check if req is acceptable */ 1422 ret = _setup_req(1, thrd, idx, &xs); 1423 if (ret < 0) 1424 goto xfer_exit; 1425 1426 if (ret > pl330->mcbufsz / 2) { 1427 dev_info(pl330->ddma.dev, "%s:%d Trying increasing mcbufsz\n", 1428 __func__, __LINE__); 1429 ret = -ENOMEM; 1430 goto xfer_exit; 1431 } 1432 1433 /* Hook the request */ 1434 thrd->lstenq = idx; 1435 thrd->req[idx].desc = desc; 1436 _setup_req(0, thrd, idx, &xs); 1437 1438 ret = 0; 1439 1440xfer_exit: 1441 spin_unlock_irqrestore(&pl330->lock, flags); 1442 1443 return ret; 1444} 1445 1446static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err) 1447{ 1448 struct dma_pl330_chan *pch; 1449 unsigned long flags; 1450 1451 if (!desc) 1452 return; 1453 1454 pch = desc->pchan; 1455 1456 /* If desc aborted */ 1457 if (!pch) 1458 return; 1459 1460 spin_lock_irqsave(&pch->lock, flags); 1461 1462 desc->status = DONE; 1463 1464 spin_unlock_irqrestore(&pch->lock, flags); 1465 1466 tasklet_schedule(&pch->task); 1467} 1468 1469static void pl330_dotask(unsigned long data) 1470{ 1471 struct pl330_dmac *pl330 = (struct pl330_dmac *) data; 1472 unsigned long flags; 1473 int i; 1474 1475 spin_lock_irqsave(&pl330->lock, flags); 1476 1477 /* The DMAC itself gone nuts */ 1478 if (pl330->dmac_tbd.reset_dmac) { 1479 pl330->state = DYING; 1480 /* Reset the manager too */ 1481 pl330->dmac_tbd.reset_mngr = true; 1482 /* Clear the reset flag */ 1483 pl330->dmac_tbd.reset_dmac = false; 1484 } 1485 1486 if (pl330->dmac_tbd.reset_mngr) { 1487 _stop(pl330->manager); 1488 /* Reset all channels */ 1489 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1; 1490 /* Clear the reset flag */ 1491 pl330->dmac_tbd.reset_mngr = false; 1492 } 1493 1494 for (i = 0; i < pl330->pcfg.num_chan; i++) { 1495 1496 if (pl330->dmac_tbd.reset_chan & (1 << i)) { 1497 struct pl330_thread *thrd = &pl330->channels[i]; 1498 void __iomem *regs = pl330->base; 1499 enum pl330_op_err err; 1500 1501 _stop(thrd); 1502 1503 if (readl(regs + FSC) & (1 << thrd->id)) 1504 err = PL330_ERR_FAIL; 1505 else 1506 err = PL330_ERR_ABORT; 1507 1508 spin_unlock_irqrestore(&pl330->lock, flags); 1509 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err); 1510 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err); 1511 spin_lock_irqsave(&pl330->lock, flags); 1512 1513 thrd->req[0].desc = NULL; 1514 thrd->req[1].desc = NULL; 1515 thrd->req_running = -1; 1516 1517 /* Clear the reset flag */ 1518 pl330->dmac_tbd.reset_chan &= ~(1 << i); 1519 } 1520 } 1521 1522 spin_unlock_irqrestore(&pl330->lock, flags); 1523 1524 return; 1525} 1526 1527/* Returns 1 if state was updated, 0 otherwise */ 1528static int pl330_update(struct pl330_dmac *pl330) 1529{ 1530 struct dma_pl330_desc *descdone, *tmp; 1531 unsigned long flags; 1532 void __iomem *regs; 1533 u32 val; 1534 int id, ev, ret = 0; 1535 1536 regs = pl330->base; 1537 1538 spin_lock_irqsave(&pl330->lock, flags); 1539 1540 val = readl(regs + FSM) & 0x1; 1541 if (val) 1542 pl330->dmac_tbd.reset_mngr = true; 1543 else 1544 pl330->dmac_tbd.reset_mngr = false; 1545 1546 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1); 1547 pl330->dmac_tbd.reset_chan |= val; 1548 if (val) { 1549 int i = 0; 1550 while (i < pl330->pcfg.num_chan) { 1551 if (val & (1 << i)) { 1552 dev_info(pl330->ddma.dev, 1553 "Reset Channel-%d\t CS-%x FTC-%x\n", 1554 i, readl(regs + CS(i)), 1555 readl(regs + FTC(i))); 1556 _stop(&pl330->channels[i]); 1557 } 1558 i++; 1559 } 1560 } 1561 1562 /* Check which event happened i.e, thread notified */ 1563 val = readl(regs + ES); 1564 if (pl330->pcfg.num_events < 32 1565 && val & ~((1 << pl330->pcfg.num_events) - 1)) { 1566 pl330->dmac_tbd.reset_dmac = true; 1567 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__, 1568 __LINE__); 1569 ret = 1; 1570 goto updt_exit; 1571 } 1572 1573 for (ev = 0; ev < pl330->pcfg.num_events; ev++) { 1574 if (val & (1 << ev)) { /* Event occurred */ 1575 struct pl330_thread *thrd; 1576 u32 inten = readl(regs + INTEN); 1577 int active; 1578 1579 /* Clear the event */ 1580 if (inten & (1 << ev)) 1581 writel(1 << ev, regs + INTCLR); 1582 1583 ret = 1; 1584 1585 id = pl330->events[ev]; 1586 1587 thrd = &pl330->channels[id]; 1588 1589 active = thrd->req_running; 1590 if (active == -1) /* Aborted */ 1591 continue; 1592 1593 /* Detach the req */ 1594 descdone = thrd->req[active].desc; 1595 thrd->req[active].desc = NULL; 1596 1597 thrd->req_running = -1; 1598 1599 /* Get going again ASAP */ 1600 _start(thrd); 1601 1602 /* For now, just make a list of callbacks to be done */ 1603 list_add_tail(&descdone->rqd, &pl330->req_done); 1604 } 1605 } 1606 1607 /* Now that we are in no hurry, do the callbacks */ 1608 list_for_each_entry_safe(descdone, tmp, &pl330->req_done, rqd) { 1609 list_del(&descdone->rqd); 1610 spin_unlock_irqrestore(&pl330->lock, flags); 1611 dma_pl330_rqcb(descdone, PL330_ERR_NONE); 1612 spin_lock_irqsave(&pl330->lock, flags); 1613 } 1614 1615updt_exit: 1616 spin_unlock_irqrestore(&pl330->lock, flags); 1617 1618 if (pl330->dmac_tbd.reset_dmac 1619 || pl330->dmac_tbd.reset_mngr 1620 || pl330->dmac_tbd.reset_chan) { 1621 ret = 1; 1622 tasklet_schedule(&pl330->tasks); 1623 } 1624 1625 return ret; 1626} 1627 1628/* Reserve an event */ 1629static inline int _alloc_event(struct pl330_thread *thrd) 1630{ 1631 struct pl330_dmac *pl330 = thrd->dmac; 1632 int ev; 1633 1634 for (ev = 0; ev < pl330->pcfg.num_events; ev++) 1635 if (pl330->events[ev] == -1) { 1636 pl330->events[ev] = thrd->id; 1637 return ev; 1638 } 1639 1640 return -1; 1641} 1642 1643static bool _chan_ns(const struct pl330_dmac *pl330, int i) 1644{ 1645 return pl330->pcfg.irq_ns & (1 << i); 1646} 1647 1648/* Upon success, returns IdentityToken for the 1649 * allocated channel, NULL otherwise. 1650 */ 1651static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330) 1652{ 1653 struct pl330_thread *thrd = NULL; 1654 unsigned long flags; 1655 int chans, i; 1656 1657 if (pl330->state == DYING) 1658 return NULL; 1659 1660 chans = pl330->pcfg.num_chan; 1661 1662 spin_lock_irqsave(&pl330->lock, flags); 1663 1664 for (i = 0; i < chans; i++) { 1665 thrd = &pl330->channels[i]; 1666 if ((thrd->free) && (!_manager_ns(thrd) || 1667 _chan_ns(pl330, i))) { 1668 thrd->ev = _alloc_event(thrd); 1669 if (thrd->ev >= 0) { 1670 thrd->free = false; 1671 thrd->lstenq = 1; 1672 thrd->req[0].desc = NULL; 1673 thrd->req[1].desc = NULL; 1674 thrd->req_running = -1; 1675 break; 1676 } 1677 } 1678 thrd = NULL; 1679 } 1680 1681 spin_unlock_irqrestore(&pl330->lock, flags); 1682 1683 return thrd; 1684} 1685 1686/* Release an event */ 1687static inline void _free_event(struct pl330_thread *thrd, int ev) 1688{ 1689 struct pl330_dmac *pl330 = thrd->dmac; 1690 1691 /* If the event is valid and was held by the thread */ 1692 if (ev >= 0 && ev < pl330->pcfg.num_events 1693 && pl330->events[ev] == thrd->id) 1694 pl330->events[ev] = -1; 1695} 1696 1697static void pl330_release_channel(struct pl330_thread *thrd) 1698{ 1699 struct pl330_dmac *pl330; 1700 unsigned long flags; 1701 1702 if (!thrd || thrd->free) 1703 return; 1704 1705 _stop(thrd); 1706 1707 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT); 1708 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT); 1709 1710 pl330 = thrd->dmac; 1711 1712 spin_lock_irqsave(&pl330->lock, flags); 1713 _free_event(thrd, thrd->ev); 1714 thrd->free = true; 1715 spin_unlock_irqrestore(&pl330->lock, flags); 1716} 1717 1718/* Initialize the structure for PL330 configuration, that can be used 1719 * by the client driver the make best use of the DMAC 1720 */ 1721static void read_dmac_config(struct pl330_dmac *pl330) 1722{ 1723 void __iomem *regs = pl330->base; 1724 u32 val; 1725 1726 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT; 1727 val &= CRD_DATA_WIDTH_MASK; 1728 pl330->pcfg.data_bus_width = 8 * (1 << val); 1729 1730 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT; 1731 val &= CRD_DATA_BUFF_MASK; 1732 pl330->pcfg.data_buf_dep = val + 1; 1733 1734 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT; 1735 val &= CR0_NUM_CHANS_MASK; 1736 val += 1; 1737 pl330->pcfg.num_chan = val; 1738 1739 val = readl(regs + CR0); 1740 if (val & CR0_PERIPH_REQ_SET) { 1741 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK; 1742 val += 1; 1743 pl330->pcfg.num_peri = val; 1744 pl330->pcfg.peri_ns = readl(regs + CR4); 1745 } else { 1746 pl330->pcfg.num_peri = 0; 1747 } 1748 1749 val = readl(regs + CR0); 1750 if (val & CR0_BOOT_MAN_NS) 1751 pl330->pcfg.mode |= DMAC_MODE_NS; 1752 else 1753 pl330->pcfg.mode &= ~DMAC_MODE_NS; 1754 1755 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT; 1756 val &= CR0_NUM_EVENTS_MASK; 1757 val += 1; 1758 pl330->pcfg.num_events = val; 1759 1760 pl330->pcfg.irq_ns = readl(regs + CR3); 1761} 1762 1763static inline void _reset_thread(struct pl330_thread *thrd) 1764{ 1765 struct pl330_dmac *pl330 = thrd->dmac; 1766 1767 thrd->req[0].mc_cpu = pl330->mcode_cpu 1768 + (thrd->id * pl330->mcbufsz); 1769 thrd->req[0].mc_bus = pl330->mcode_bus 1770 + (thrd->id * pl330->mcbufsz); 1771 thrd->req[0].desc = NULL; 1772 1773 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu 1774 + pl330->mcbufsz / 2; 1775 thrd->req[1].mc_bus = thrd->req[0].mc_bus 1776 + pl330->mcbufsz / 2; 1777 thrd->req[1].desc = NULL; 1778 1779 thrd->req_running = -1; 1780} 1781 1782static int dmac_alloc_threads(struct pl330_dmac *pl330) 1783{ 1784 int chans = pl330->pcfg.num_chan; 1785 struct pl330_thread *thrd; 1786 int i; 1787 1788 /* Allocate 1 Manager and 'chans' Channel threads */ 1789 pl330->channels = kzalloc((1 + chans) * sizeof(*thrd), 1790 GFP_KERNEL); 1791 if (!pl330->channels) 1792 return -ENOMEM; 1793 1794 /* Init Channel threads */ 1795 for (i = 0; i < chans; i++) { 1796 thrd = &pl330->channels[i]; 1797 thrd->id = i; 1798 thrd->dmac = pl330; 1799 _reset_thread(thrd); 1800 thrd->free = true; 1801 } 1802 1803 /* MANAGER is indexed at the end */ 1804 thrd = &pl330->channels[chans]; 1805 thrd->id = chans; 1806 thrd->dmac = pl330; 1807 thrd->free = false; 1808 pl330->manager = thrd; 1809 1810 return 0; 1811} 1812 1813static int dmac_alloc_resources(struct pl330_dmac *pl330) 1814{ 1815 int chans = pl330->pcfg.num_chan; 1816 int ret; 1817 1818 /* 1819 * Alloc MicroCode buffer for 'chans' Channel threads. 1820 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN) 1821 */ 1822 pl330->mcode_cpu = dma_alloc_coherent(pl330->ddma.dev, 1823 chans * pl330->mcbufsz, 1824 &pl330->mcode_bus, GFP_KERNEL); 1825 if (!pl330->mcode_cpu) { 1826 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n", 1827 __func__, __LINE__); 1828 return -ENOMEM; 1829 } 1830 1831 ret = dmac_alloc_threads(pl330); 1832 if (ret) { 1833 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n", 1834 __func__, __LINE__); 1835 dma_free_coherent(pl330->ddma.dev, 1836 chans * pl330->mcbufsz, 1837 pl330->mcode_cpu, pl330->mcode_bus); 1838 return ret; 1839 } 1840 1841 return 0; 1842} 1843 1844static int pl330_add(struct pl330_dmac *pl330) 1845{ 1846 void __iomem *regs; 1847 int i, ret; 1848 1849 regs = pl330->base; 1850 1851 /* Check if we can handle this DMAC */ 1852 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) { 1853 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n", 1854 pl330->pcfg.periph_id); 1855 return -EINVAL; 1856 } 1857 1858 /* Read the configuration of the DMAC */ 1859 read_dmac_config(pl330); 1860 1861 if (pl330->pcfg.num_events == 0) { 1862 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n", 1863 __func__, __LINE__); 1864 return -EINVAL; 1865 } 1866 1867 spin_lock_init(&pl330->lock); 1868 1869 INIT_LIST_HEAD(&pl330->req_done); 1870 1871 /* Use default MC buffer size if not provided */ 1872 if (!pl330->mcbufsz) 1873 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2; 1874 1875 /* Mark all events as free */ 1876 for (i = 0; i < pl330->pcfg.num_events; i++) 1877 pl330->events[i] = -1; 1878 1879 /* Allocate resources needed by the DMAC */ 1880 ret = dmac_alloc_resources(pl330); 1881 if (ret) { 1882 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n"); 1883 return ret; 1884 } 1885 1886 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330); 1887 1888 pl330->state = INIT; 1889 1890 return 0; 1891} 1892 1893static int dmac_free_threads(struct pl330_dmac *pl330) 1894{ 1895 struct pl330_thread *thrd; 1896 int i; 1897 1898 /* Release Channel threads */ 1899 for (i = 0; i < pl330->pcfg.num_chan; i++) { 1900 thrd = &pl330->channels[i]; 1901 pl330_release_channel(thrd); 1902 } 1903 1904 /* Free memory */ 1905 kfree(pl330->channels); 1906 1907 return 0; 1908} 1909 1910static void pl330_del(struct pl330_dmac *pl330) 1911{ 1912 pl330->state = UNINIT; 1913 1914 tasklet_kill(&pl330->tasks); 1915 1916 /* Free DMAC resources */ 1917 dmac_free_threads(pl330); 1918 1919 dma_free_coherent(pl330->ddma.dev, 1920 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu, 1921 pl330->mcode_bus); 1922} 1923 1924/* forward declaration */ 1925static struct amba_driver pl330_driver; 1926 1927static inline struct dma_pl330_chan * 1928to_pchan(struct dma_chan *ch) 1929{ 1930 if (!ch) 1931 return NULL; 1932 1933 return container_of(ch, struct dma_pl330_chan, chan); 1934} 1935 1936static inline struct dma_pl330_desc * 1937to_desc(struct dma_async_tx_descriptor *tx) 1938{ 1939 return container_of(tx, struct dma_pl330_desc, txd); 1940} 1941 1942static inline void fill_queue(struct dma_pl330_chan *pch) 1943{ 1944 struct dma_pl330_desc *desc; 1945 int ret; 1946 1947 list_for_each_entry(desc, &pch->work_list, node) { 1948 1949 /* If already submitted */ 1950 if (desc->status == BUSY) 1951 continue; 1952 1953 ret = pl330_submit_req(pch->thread, desc); 1954 if (!ret) { 1955 desc->status = BUSY; 1956 } else if (ret == -EAGAIN) { 1957 /* QFull or DMAC Dying */ 1958 break; 1959 } else { 1960 /* Unacceptable request */ 1961 desc->status = DONE; 1962 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n", 1963 __func__, __LINE__, desc->txd.cookie); 1964 tasklet_schedule(&pch->task); 1965 } 1966 } 1967} 1968 1969static void pl330_tasklet(unsigned long data) 1970{ 1971 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data; 1972 struct dma_pl330_desc *desc, *_dt; 1973 unsigned long flags; 1974 bool power_down = false; 1975 1976 spin_lock_irqsave(&pch->lock, flags); 1977 1978 /* Pick up ripe tomatoes */ 1979 list_for_each_entry_safe(desc, _dt, &pch->work_list, node) 1980 if (desc->status == DONE) { 1981 if (!pch->cyclic) 1982 dma_cookie_complete(&desc->txd); 1983 list_move_tail(&desc->node, &pch->completed_list); 1984 } 1985 1986 /* Try to submit a req imm. next to the last completed cookie */ 1987 fill_queue(pch); 1988 1989 if (list_empty(&pch->work_list)) { 1990 spin_lock(&pch->thread->dmac->lock); 1991 _stop(pch->thread); 1992 spin_unlock(&pch->thread->dmac->lock); 1993 power_down = true; 1994 } else { 1995 /* Make sure the PL330 Channel thread is active */ 1996 spin_lock(&pch->thread->dmac->lock); 1997 _start(pch->thread); 1998 spin_unlock(&pch->thread->dmac->lock); 1999 } 2000 2001 while (!list_empty(&pch->completed_list)) { 2002 dma_async_tx_callback callback; 2003 void *callback_param; 2004 2005 desc = list_first_entry(&pch->completed_list, 2006 struct dma_pl330_desc, node); 2007 2008 callback = desc->txd.callback; 2009 callback_param = desc->txd.callback_param; 2010 2011 if (pch->cyclic) { 2012 desc->status = PREP; 2013 list_move_tail(&desc->node, &pch->work_list); 2014 if (power_down) { 2015 spin_lock(&pch->thread->dmac->lock); 2016 _start(pch->thread); 2017 spin_unlock(&pch->thread->dmac->lock); 2018 power_down = false; 2019 } 2020 } else { 2021 desc->status = FREE; 2022 list_move_tail(&desc->node, &pch->dmac->desc_pool); 2023 } 2024 2025 dma_descriptor_unmap(&desc->txd); 2026 2027 if (callback) { 2028 spin_unlock_irqrestore(&pch->lock, flags); 2029 callback(callback_param); 2030 spin_lock_irqsave(&pch->lock, flags); 2031 } 2032 } 2033 spin_unlock_irqrestore(&pch->lock, flags); 2034 2035 /* If work list empty, power down */ 2036 if (power_down) { 2037 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); 2038 pm_runtime_put_autosuspend(pch->dmac->ddma.dev); 2039 } 2040} 2041 2042bool pl330_filter(struct dma_chan *chan, void *param) 2043{ 2044 u8 *peri_id; 2045 2046 if (chan->device->dev->driver != &pl330_driver.drv) 2047 return false; 2048 2049 peri_id = chan->private; 2050 return *peri_id == (unsigned long)param; 2051} 2052EXPORT_SYMBOL(pl330_filter); 2053 2054static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec, 2055 struct of_dma *ofdma) 2056{ 2057 int count = dma_spec->args_count; 2058 struct pl330_dmac *pl330 = ofdma->of_dma_data; 2059 unsigned int chan_id; 2060 2061 if (!pl330) 2062 return NULL; 2063 2064 if (count != 1) 2065 return NULL; 2066 2067 chan_id = dma_spec->args[0]; 2068 if (chan_id >= pl330->num_peripherals) 2069 return NULL; 2070 2071 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan); 2072} 2073 2074static int pl330_alloc_chan_resources(struct dma_chan *chan) 2075{ 2076 struct dma_pl330_chan *pch = to_pchan(chan); 2077 struct pl330_dmac *pl330 = pch->dmac; 2078 unsigned long flags; 2079 2080 spin_lock_irqsave(&pch->lock, flags); 2081 2082 dma_cookie_init(chan); 2083 pch->cyclic = false; 2084 2085 pch->thread = pl330_request_channel(pl330); 2086 if (!pch->thread) { 2087 spin_unlock_irqrestore(&pch->lock, flags); 2088 return -ENOMEM; 2089 } 2090 2091 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch); 2092 2093 spin_unlock_irqrestore(&pch->lock, flags); 2094 2095 return 1; 2096} 2097 2098static int pl330_config(struct dma_chan *chan, 2099 struct dma_slave_config *slave_config) 2100{ 2101 struct dma_pl330_chan *pch = to_pchan(chan); 2102 2103 if (slave_config->direction == DMA_MEM_TO_DEV) { 2104 if (slave_config->dst_addr) 2105 pch->fifo_addr = slave_config->dst_addr; 2106 if (slave_config->dst_addr_width) 2107 pch->burst_sz = __ffs(slave_config->dst_addr_width); 2108 if (slave_config->dst_maxburst) 2109 pch->burst_len = slave_config->dst_maxburst; 2110 } else if (slave_config->direction == DMA_DEV_TO_MEM) { 2111 if (slave_config->src_addr) 2112 pch->fifo_addr = slave_config->src_addr; 2113 if (slave_config->src_addr_width) 2114 pch->burst_sz = __ffs(slave_config->src_addr_width); 2115 if (slave_config->src_maxburst) 2116 pch->burst_len = slave_config->src_maxburst; 2117 } 2118 2119 return 0; 2120} 2121 2122static int pl330_terminate_all(struct dma_chan *chan) 2123{ 2124 struct dma_pl330_chan *pch = to_pchan(chan); 2125 struct dma_pl330_desc *desc; 2126 unsigned long flags; 2127 struct pl330_dmac *pl330 = pch->dmac; 2128 LIST_HEAD(list); 2129 2130 pm_runtime_get_sync(pl330->ddma.dev); 2131 spin_lock_irqsave(&pch->lock, flags); 2132 spin_lock(&pl330->lock); 2133 _stop(pch->thread); 2134 spin_unlock(&pl330->lock); 2135 2136 pch->thread->req[0].desc = NULL; 2137 pch->thread->req[1].desc = NULL; 2138 pch->thread->req_running = -1; 2139 2140 /* Mark all desc done */ 2141 list_for_each_entry(desc, &pch->submitted_list, node) { 2142 desc->status = FREE; 2143 dma_cookie_complete(&desc->txd); 2144 } 2145 2146 list_for_each_entry(desc, &pch->work_list , node) { 2147 desc->status = FREE; 2148 dma_cookie_complete(&desc->txd); 2149 } 2150 2151 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool); 2152 list_splice_tail_init(&pch->work_list, &pl330->desc_pool); 2153 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool); 2154 spin_unlock_irqrestore(&pch->lock, flags); 2155 pm_runtime_mark_last_busy(pl330->ddma.dev); 2156 pm_runtime_put_autosuspend(pl330->ddma.dev); 2157 2158 return 0; 2159} 2160 2161/* 2162 * We don't support DMA_RESUME command because of hardware 2163 * limitations, so after pausing the channel we cannot restore 2164 * it to active state. We have to terminate channel and setup 2165 * DMA transfer again. This pause feature was implemented to 2166 * allow safely read residue before channel termination. 2167 */ 2168static int pl330_pause(struct dma_chan *chan) 2169{ 2170 struct dma_pl330_chan *pch = to_pchan(chan); 2171 struct pl330_dmac *pl330 = pch->dmac; 2172 unsigned long flags; 2173 2174 pm_runtime_get_sync(pl330->ddma.dev); 2175 spin_lock_irqsave(&pch->lock, flags); 2176 2177 spin_lock(&pl330->lock); 2178 _stop(pch->thread); 2179 spin_unlock(&pl330->lock); 2180 2181 spin_unlock_irqrestore(&pch->lock, flags); 2182 pm_runtime_mark_last_busy(pl330->ddma.dev); 2183 pm_runtime_put_autosuspend(pl330->ddma.dev); 2184 2185 return 0; 2186} 2187 2188static void pl330_free_chan_resources(struct dma_chan *chan) 2189{ 2190 struct dma_pl330_chan *pch = to_pchan(chan); 2191 unsigned long flags; 2192 2193 tasklet_kill(&pch->task); 2194 2195 pm_runtime_get_sync(pch->dmac->ddma.dev); 2196 spin_lock_irqsave(&pch->lock, flags); 2197 2198 pl330_release_channel(pch->thread); 2199 pch->thread = NULL; 2200 2201 if (pch->cyclic) 2202 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); 2203 2204 spin_unlock_irqrestore(&pch->lock, flags); 2205 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); 2206 pm_runtime_put_autosuspend(pch->dmac->ddma.dev); 2207} 2208 2209static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch, 2210 struct dma_pl330_desc *desc) 2211{ 2212 struct pl330_thread *thrd = pch->thread; 2213 struct pl330_dmac *pl330 = pch->dmac; 2214 void __iomem *regs = thrd->dmac->base; 2215 u32 val, addr; 2216 2217 pm_runtime_get_sync(pl330->ddma.dev); 2218 val = addr = 0; 2219 if (desc->rqcfg.src_inc) { 2220 val = readl(regs + SA(thrd->id)); 2221 addr = desc->px.src_addr; 2222 } else { 2223 val = readl(regs + DA(thrd->id)); 2224 addr = desc->px.dst_addr; 2225 } 2226 pm_runtime_mark_last_busy(pch->dmac->ddma.dev); 2227 pm_runtime_put_autosuspend(pl330->ddma.dev); 2228 return val - addr; 2229} 2230 2231static enum dma_status 2232pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 2233 struct dma_tx_state *txstate) 2234{ 2235 enum dma_status ret; 2236 unsigned long flags; 2237 struct dma_pl330_desc *desc, *running = NULL; 2238 struct dma_pl330_chan *pch = to_pchan(chan); 2239 unsigned int transferred, residual = 0; 2240 2241 ret = dma_cookie_status(chan, cookie, txstate); 2242 2243 if (!txstate) 2244 return ret; 2245 2246 if (ret == DMA_COMPLETE) 2247 goto out; 2248 2249 spin_lock_irqsave(&pch->lock, flags); 2250 2251 if (pch->thread->req_running != -1) 2252 running = pch->thread->req[pch->thread->req_running].desc; 2253 2254 /* Check in pending list */ 2255 list_for_each_entry(desc, &pch->work_list, node) { 2256 if (desc->status == DONE) 2257 transferred = desc->bytes_requested; 2258 else if (running && desc == running) 2259 transferred = 2260 pl330_get_current_xferred_count(pch, desc); 2261 else 2262 transferred = 0; 2263 residual += desc->bytes_requested - transferred; 2264 if (desc->txd.cookie == cookie) { 2265 switch (desc->status) { 2266 case DONE: 2267 ret = DMA_COMPLETE; 2268 break; 2269 case PREP: 2270 case BUSY: 2271 ret = DMA_IN_PROGRESS; 2272 break; 2273 default: 2274 WARN_ON(1); 2275 } 2276 break; 2277 } 2278 if (desc->last) 2279 residual = 0; 2280 } 2281 spin_unlock_irqrestore(&pch->lock, flags); 2282 2283out: 2284 dma_set_residue(txstate, residual); 2285 2286 return ret; 2287} 2288 2289static void pl330_issue_pending(struct dma_chan *chan) 2290{ 2291 struct dma_pl330_chan *pch = to_pchan(chan); 2292 unsigned long flags; 2293 2294 spin_lock_irqsave(&pch->lock, flags); 2295 if (list_empty(&pch->work_list)) { 2296 /* 2297 * Warn on nothing pending. Empty submitted_list may 2298 * break our pm_runtime usage counter as it is 2299 * updated on work_list emptiness status. 2300 */ 2301 WARN_ON(list_empty(&pch->submitted_list)); 2302 pm_runtime_get_sync(pch->dmac->ddma.dev); 2303 } 2304 list_splice_tail_init(&pch->submitted_list, &pch->work_list); 2305 spin_unlock_irqrestore(&pch->lock, flags); 2306 2307 pl330_tasklet((unsigned long)pch); 2308} 2309 2310/* 2311 * We returned the last one of the circular list of descriptor(s) 2312 * from prep_xxx, so the argument to submit corresponds to the last 2313 * descriptor of the list. 2314 */ 2315static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) 2316{ 2317 struct dma_pl330_desc *desc, *last = to_desc(tx); 2318 struct dma_pl330_chan *pch = to_pchan(tx->chan); 2319 dma_cookie_t cookie; 2320 unsigned long flags; 2321 2322 spin_lock_irqsave(&pch->lock, flags); 2323 2324 /* Assign cookies to all nodes */ 2325 while (!list_empty(&last->node)) { 2326 desc = list_entry(last->node.next, struct dma_pl330_desc, node); 2327 if (pch->cyclic) { 2328 desc->txd.callback = last->txd.callback; 2329 desc->txd.callback_param = last->txd.callback_param; 2330 } 2331 desc->last = false; 2332 2333 dma_cookie_assign(&desc->txd); 2334 2335 list_move_tail(&desc->node, &pch->submitted_list); 2336 } 2337 2338 last->last = true; 2339 cookie = dma_cookie_assign(&last->txd); 2340 list_add_tail(&last->node, &pch->submitted_list); 2341 spin_unlock_irqrestore(&pch->lock, flags); 2342 2343 return cookie; 2344} 2345 2346static inline void _init_desc(struct dma_pl330_desc *desc) 2347{ 2348 desc->rqcfg.swap = SWAP_NO; 2349 desc->rqcfg.scctl = CCTRL0; 2350 desc->rqcfg.dcctl = CCTRL0; 2351 desc->txd.tx_submit = pl330_tx_submit; 2352 2353 INIT_LIST_HEAD(&desc->node); 2354} 2355 2356/* Returns the number of descriptors added to the DMAC pool */ 2357static int add_desc(struct pl330_dmac *pl330, gfp_t flg, int count) 2358{ 2359 struct dma_pl330_desc *desc; 2360 unsigned long flags; 2361 int i; 2362 2363 desc = kcalloc(count, sizeof(*desc), flg); 2364 if (!desc) 2365 return 0; 2366 2367 spin_lock_irqsave(&pl330->pool_lock, flags); 2368 2369 for (i = 0; i < count; i++) { 2370 _init_desc(&desc[i]); 2371 list_add_tail(&desc[i].node, &pl330->desc_pool); 2372 } 2373 2374 spin_unlock_irqrestore(&pl330->pool_lock, flags); 2375 2376 return count; 2377} 2378 2379static struct dma_pl330_desc *pluck_desc(struct pl330_dmac *pl330) 2380{ 2381 struct dma_pl330_desc *desc = NULL; 2382 unsigned long flags; 2383 2384 spin_lock_irqsave(&pl330->pool_lock, flags); 2385 2386 if (!list_empty(&pl330->desc_pool)) { 2387 desc = list_entry(pl330->desc_pool.next, 2388 struct dma_pl330_desc, node); 2389 2390 list_del_init(&desc->node); 2391 2392 desc->status = PREP; 2393 desc->txd.callback = NULL; 2394 } 2395 2396 spin_unlock_irqrestore(&pl330->pool_lock, flags); 2397 2398 return desc; 2399} 2400 2401static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) 2402{ 2403 struct pl330_dmac *pl330 = pch->dmac; 2404 u8 *peri_id = pch->chan.private; 2405 struct dma_pl330_desc *desc; 2406 2407 /* Pluck one desc from the pool of DMAC */ 2408 desc = pluck_desc(pl330); 2409 2410 /* If the DMAC pool is empty, alloc new */ 2411 if (!desc) { 2412 if (!add_desc(pl330, GFP_ATOMIC, 1)) 2413 return NULL; 2414 2415 /* Try again */ 2416 desc = pluck_desc(pl330); 2417 if (!desc) { 2418 dev_err(pch->dmac->ddma.dev, 2419 "%s:%d ALERT!\n", __func__, __LINE__); 2420 return NULL; 2421 } 2422 } 2423 2424 /* Initialize the descriptor */ 2425 desc->pchan = pch; 2426 desc->txd.cookie = 0; 2427 async_tx_ack(&desc->txd); 2428 2429 desc->peri = peri_id ? pch->chan.chan_id : 0; 2430 desc->rqcfg.pcfg = &pch->dmac->pcfg; 2431 2432 dma_async_tx_descriptor_init(&desc->txd, &pch->chan); 2433 2434 return desc; 2435} 2436 2437static inline void fill_px(struct pl330_xfer *px, 2438 dma_addr_t dst, dma_addr_t src, size_t len) 2439{ 2440 px->bytes = len; 2441 px->dst_addr = dst; 2442 px->src_addr = src; 2443} 2444 2445static struct dma_pl330_desc * 2446__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst, 2447 dma_addr_t src, size_t len) 2448{ 2449 struct dma_pl330_desc *desc = pl330_get_desc(pch); 2450 2451 if (!desc) { 2452 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", 2453 __func__, __LINE__); 2454 return NULL; 2455 } 2456 2457 /* 2458 * Ideally we should lookout for reqs bigger than 2459 * those that can be programmed with 256 bytes of 2460 * MC buffer, but considering a req size is seldom 2461 * going to be word-unaligned and more than 200MB, 2462 * we take it easy. 2463 * Also, should the limit is reached we'd rather 2464 * have the platform increase MC buffer size than 2465 * complicating this API driver. 2466 */ 2467 fill_px(&desc->px, dst, src, len); 2468 2469 return desc; 2470} 2471 2472/* Call after fixing burst size */ 2473static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) 2474{ 2475 struct dma_pl330_chan *pch = desc->pchan; 2476 struct pl330_dmac *pl330 = pch->dmac; 2477 int burst_len; 2478 2479 burst_len = pl330->pcfg.data_bus_width / 8; 2480 burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan; 2481 burst_len >>= desc->rqcfg.brst_size; 2482 2483 /* src/dst_burst_len can't be more than 16 */ 2484 if (burst_len > 16) 2485 burst_len = 16; 2486 2487 while (burst_len > 1) { 2488 if (!(len % (burst_len << desc->rqcfg.brst_size))) 2489 break; 2490 burst_len--; 2491 } 2492 2493 return burst_len; 2494} 2495 2496static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( 2497 struct dma_chan *chan, dma_addr_t dma_addr, size_t len, 2498 size_t period_len, enum dma_transfer_direction direction, 2499 unsigned long flags) 2500{ 2501 struct dma_pl330_desc *desc = NULL, *first = NULL; 2502 struct dma_pl330_chan *pch = to_pchan(chan); 2503 struct pl330_dmac *pl330 = pch->dmac; 2504 unsigned int i; 2505 dma_addr_t dst; 2506 dma_addr_t src; 2507 2508 if (len % period_len != 0) 2509 return NULL; 2510 2511 if (!is_slave_direction(direction)) { 2512 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n", 2513 __func__, __LINE__); 2514 return NULL; 2515 } 2516 2517 for (i = 0; i < len / period_len; i++) { 2518 desc = pl330_get_desc(pch); 2519 if (!desc) { 2520 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n", 2521 __func__, __LINE__); 2522 2523 if (!first) 2524 return NULL; 2525 2526 spin_lock_irqsave(&pl330->pool_lock, flags); 2527 2528 while (!list_empty(&first->node)) { 2529 desc = list_entry(first->node.next, 2530 struct dma_pl330_desc, node); 2531 list_move_tail(&desc->node, &pl330->desc_pool); 2532 } 2533 2534 list_move_tail(&first->node, &pl330->desc_pool); 2535 2536 spin_unlock_irqrestore(&pl330->pool_lock, flags); 2537 2538 return NULL; 2539 } 2540 2541 switch (direction) { 2542 case DMA_MEM_TO_DEV: 2543 desc->rqcfg.src_inc = 1; 2544 desc->rqcfg.dst_inc = 0; 2545 src = dma_addr; 2546 dst = pch->fifo_addr; 2547 break; 2548 case DMA_DEV_TO_MEM: 2549 desc->rqcfg.src_inc = 0; 2550 desc->rqcfg.dst_inc = 1; 2551 src = pch->fifo_addr; 2552 dst = dma_addr; 2553 break; 2554 default: 2555 break; 2556 } 2557 2558 desc->rqtype = direction; 2559 desc->rqcfg.brst_size = pch->burst_sz; 2560 desc->rqcfg.brst_len = 1; 2561 desc->bytes_requested = period_len; 2562 fill_px(&desc->px, dst, src, period_len); 2563 2564 if (!first) 2565 first = desc; 2566 else 2567 list_add_tail(&desc->node, &first->node); 2568 2569 dma_addr += period_len; 2570 } 2571 2572 if (!desc) 2573 return NULL; 2574 2575 pch->cyclic = true; 2576 desc->txd.flags = flags; 2577 2578 return &desc->txd; 2579} 2580 2581static struct dma_async_tx_descriptor * 2582pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, 2583 dma_addr_t src, size_t len, unsigned long flags) 2584{ 2585 struct dma_pl330_desc *desc; 2586 struct dma_pl330_chan *pch = to_pchan(chan); 2587 struct pl330_dmac *pl330 = pch->dmac; 2588 int burst; 2589 2590 if (unlikely(!pch || !len)) 2591 return NULL; 2592 2593 desc = __pl330_prep_dma_memcpy(pch, dst, src, len); 2594 if (!desc) 2595 return NULL; 2596 2597 desc->rqcfg.src_inc = 1; 2598 desc->rqcfg.dst_inc = 1; 2599 desc->rqtype = DMA_MEM_TO_MEM; 2600 2601 /* Select max possible burst size */ 2602 burst = pl330->pcfg.data_bus_width / 8; 2603 2604 /* 2605 * Make sure we use a burst size that aligns with all the memcpy 2606 * parameters because our DMA programming algorithm doesn't cope with 2607 * transfers which straddle an entry in the DMA device's MFIFO. 2608 */ 2609 while ((src | dst | len) & (burst - 1)) 2610 burst /= 2; 2611 2612 desc->rqcfg.brst_size = 0; 2613 while (burst != (1 << desc->rqcfg.brst_size)) 2614 desc->rqcfg.brst_size++; 2615 2616 /* 2617 * If burst size is smaller than bus width then make sure we only 2618 * transfer one at a time to avoid a burst stradling an MFIFO entry. 2619 */ 2620 if (desc->rqcfg.brst_size * 8 < pl330->pcfg.data_bus_width) 2621 desc->rqcfg.brst_len = 1; 2622 2623 desc->rqcfg.brst_len = get_burst_len(desc, len); 2624 desc->bytes_requested = len; 2625 2626 desc->txd.flags = flags; 2627 2628 return &desc->txd; 2629} 2630 2631static void __pl330_giveback_desc(struct pl330_dmac *pl330, 2632 struct dma_pl330_desc *first) 2633{ 2634 unsigned long flags; 2635 struct dma_pl330_desc *desc; 2636 2637 if (!first) 2638 return; 2639 2640 spin_lock_irqsave(&pl330->pool_lock, flags); 2641 2642 while (!list_empty(&first->node)) { 2643 desc = list_entry(first->node.next, 2644 struct dma_pl330_desc, node); 2645 list_move_tail(&desc->node, &pl330->desc_pool); 2646 } 2647 2648 list_move_tail(&first->node, &pl330->desc_pool); 2649 2650 spin_unlock_irqrestore(&pl330->pool_lock, flags); 2651} 2652 2653static struct dma_async_tx_descriptor * 2654pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2655 unsigned int sg_len, enum dma_transfer_direction direction, 2656 unsigned long flg, void *context) 2657{ 2658 struct dma_pl330_desc *first, *desc = NULL; 2659 struct dma_pl330_chan *pch = to_pchan(chan); 2660 struct scatterlist *sg; 2661 int i; 2662 dma_addr_t addr; 2663 2664 if (unlikely(!pch || !sgl || !sg_len)) 2665 return NULL; 2666 2667 addr = pch->fifo_addr; 2668 2669 first = NULL; 2670 2671 for_each_sg(sgl, sg, sg_len, i) { 2672 2673 desc = pl330_get_desc(pch); 2674 if (!desc) { 2675 struct pl330_dmac *pl330 = pch->dmac; 2676 2677 dev_err(pch->dmac->ddma.dev, 2678 "%s:%d Unable to fetch desc\n", 2679 __func__, __LINE__); 2680 __pl330_giveback_desc(pl330, first); 2681 2682 return NULL; 2683 } 2684 2685 if (!first) 2686 first = desc; 2687 else 2688 list_add_tail(&desc->node, &first->node); 2689 2690 if (direction == DMA_MEM_TO_DEV) { 2691 desc->rqcfg.src_inc = 1; 2692 desc->rqcfg.dst_inc = 0; 2693 fill_px(&desc->px, 2694 addr, sg_dma_address(sg), sg_dma_len(sg)); 2695 } else { 2696 desc->rqcfg.src_inc = 0; 2697 desc->rqcfg.dst_inc = 1; 2698 fill_px(&desc->px, 2699 sg_dma_address(sg), addr, sg_dma_len(sg)); 2700 } 2701 2702 desc->rqcfg.brst_size = pch->burst_sz; 2703 desc->rqcfg.brst_len = 1; 2704 desc->rqtype = direction; 2705 desc->bytes_requested = sg_dma_len(sg); 2706 } 2707 2708 /* Return the last desc in the chain */ 2709 desc->txd.flags = flg; 2710 return &desc->txd; 2711} 2712 2713static irqreturn_t pl330_irq_handler(int irq, void *data) 2714{ 2715 if (pl330_update(data)) 2716 return IRQ_HANDLED; 2717 else 2718 return IRQ_NONE; 2719} 2720 2721#define PL330_DMA_BUSWIDTHS \ 2722 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 2723 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 2724 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 2725 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 2726 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) 2727 2728/* 2729 * Runtime PM callbacks are provided by amba/bus.c driver. 2730 * 2731 * It is assumed here that IRQ safe runtime PM is chosen in probe and amba 2732 * bus driver will only disable/enable the clock in runtime PM callbacks. 2733 */ 2734static int __maybe_unused pl330_suspend(struct device *dev) 2735{ 2736 struct amba_device *pcdev = to_amba_device(dev); 2737 2738 pm_runtime_disable(dev); 2739 2740 if (!pm_runtime_status_suspended(dev)) { 2741 /* amba did not disable the clock */ 2742 amba_pclk_disable(pcdev); 2743 } 2744 amba_pclk_unprepare(pcdev); 2745 2746 return 0; 2747} 2748 2749static int __maybe_unused pl330_resume(struct device *dev) 2750{ 2751 struct amba_device *pcdev = to_amba_device(dev); 2752 int ret; 2753 2754 ret = amba_pclk_prepare(pcdev); 2755 if (ret) 2756 return ret; 2757 2758 if (!pm_runtime_status_suspended(dev)) 2759 ret = amba_pclk_enable(pcdev); 2760 2761 pm_runtime_enable(dev); 2762 2763 return ret; 2764} 2765 2766static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume); 2767 2768static int 2769pl330_probe(struct amba_device *adev, const struct amba_id *id) 2770{ 2771 struct dma_pl330_platdata *pdat; 2772 struct pl330_config *pcfg; 2773 struct pl330_dmac *pl330; 2774 struct dma_pl330_chan *pch, *_p; 2775 struct dma_device *pd; 2776 struct resource *res; 2777 int i, ret, irq; 2778 int num_chan; 2779 2780 pdat = dev_get_platdata(&adev->dev); 2781 2782 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); 2783 if (ret) 2784 return ret; 2785 2786 /* Allocate a new DMAC and its Channels */ 2787 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL); 2788 if (!pl330) { 2789 dev_err(&adev->dev, "unable to allocate mem\n"); 2790 return -ENOMEM; 2791 } 2792 2793 pd = &pl330->ddma; 2794 pd->dev = &adev->dev; 2795 2796 pl330->mcbufsz = pdat ? pdat->mcbuf_sz : 0; 2797 2798 res = &adev->res; 2799 pl330->base = devm_ioremap_resource(&adev->dev, res); 2800 if (IS_ERR(pl330->base)) 2801 return PTR_ERR(pl330->base); 2802 2803 amba_set_drvdata(adev, pl330); 2804 2805 for (i = 0; i < AMBA_NR_IRQS; i++) { 2806 irq = adev->irq[i]; 2807 if (irq) { 2808 ret = devm_request_irq(&adev->dev, irq, 2809 pl330_irq_handler, 0, 2810 dev_name(&adev->dev), pl330); 2811 if (ret) 2812 return ret; 2813 } else { 2814 break; 2815 } 2816 } 2817 2818 pcfg = &pl330->pcfg; 2819 2820 pcfg->periph_id = adev->periphid; 2821 ret = pl330_add(pl330); 2822 if (ret) 2823 return ret; 2824 2825 INIT_LIST_HEAD(&pl330->desc_pool); 2826 spin_lock_init(&pl330->pool_lock); 2827 2828 /* Create a descriptor pool of default size */ 2829 if (!add_desc(pl330, GFP_KERNEL, NR_DEFAULT_DESC)) 2830 dev_warn(&adev->dev, "unable to allocate desc\n"); 2831 2832 INIT_LIST_HEAD(&pd->channels); 2833 2834 /* Initialize channel parameters */ 2835 if (pdat) 2836 num_chan = max_t(int, pdat->nr_valid_peri, pcfg->num_chan); 2837 else 2838 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan); 2839 2840 pl330->num_peripherals = num_chan; 2841 2842 pl330->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL); 2843 if (!pl330->peripherals) { 2844 ret = -ENOMEM; 2845 dev_err(&adev->dev, "unable to allocate pl330->peripherals\n"); 2846 goto probe_err2; 2847 } 2848 2849 for (i = 0; i < num_chan; i++) { 2850 pch = &pl330->peripherals[i]; 2851 if (!adev->dev.of_node) 2852 pch->chan.private = pdat ? &pdat->peri_id[i] : NULL; 2853 else 2854 pch->chan.private = adev->dev.of_node; 2855 2856 INIT_LIST_HEAD(&pch->submitted_list); 2857 INIT_LIST_HEAD(&pch->work_list); 2858 INIT_LIST_HEAD(&pch->completed_list); 2859 spin_lock_init(&pch->lock); 2860 pch->thread = NULL; 2861 pch->chan.device = pd; 2862 pch->dmac = pl330; 2863 2864 /* Add the channel to the DMAC list */ 2865 list_add_tail(&pch->chan.device_node, &pd->channels); 2866 } 2867 2868 if (pdat) { 2869 pd->cap_mask = pdat->cap_mask; 2870 } else { 2871 dma_cap_set(DMA_MEMCPY, pd->cap_mask); 2872 if (pcfg->num_peri) { 2873 dma_cap_set(DMA_SLAVE, pd->cap_mask); 2874 dma_cap_set(DMA_CYCLIC, pd->cap_mask); 2875 dma_cap_set(DMA_PRIVATE, pd->cap_mask); 2876 } 2877 } 2878 2879 pd->device_alloc_chan_resources = pl330_alloc_chan_resources; 2880 pd->device_free_chan_resources = pl330_free_chan_resources; 2881 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; 2882 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic; 2883 pd->device_tx_status = pl330_tx_status; 2884 pd->device_prep_slave_sg = pl330_prep_slave_sg; 2885 pd->device_config = pl330_config; 2886 pd->device_pause = pl330_pause; 2887 pd->device_terminate_all = pl330_terminate_all; 2888 pd->device_issue_pending = pl330_issue_pending; 2889 pd->src_addr_widths = PL330_DMA_BUSWIDTHS; 2890 pd->dst_addr_widths = PL330_DMA_BUSWIDTHS; 2891 pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2892 pd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2893 2894 ret = dma_async_device_register(pd); 2895 if (ret) { 2896 dev_err(&adev->dev, "unable to register DMAC\n"); 2897 goto probe_err3; 2898 } 2899 2900 if (adev->dev.of_node) { 2901 ret = of_dma_controller_register(adev->dev.of_node, 2902 of_dma_pl330_xlate, pl330); 2903 if (ret) { 2904 dev_err(&adev->dev, 2905 "unable to register DMA to the generic DT DMA helpers\n"); 2906 } 2907 } 2908 2909 adev->dev.dma_parms = &pl330->dma_parms; 2910 2911 /* 2912 * This is the limit for transfers with a buswidth of 1, larger 2913 * buswidths will have larger limits. 2914 */ 2915 ret = dma_set_max_seg_size(&adev->dev, 1900800); 2916 if (ret) 2917 dev_err(&adev->dev, "unable to set the seg size\n"); 2918 2919 2920 dev_info(&adev->dev, 2921 "Loaded driver for PL330 DMAC-%x\n", adev->periphid); 2922 dev_info(&adev->dev, 2923 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", 2924 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan, 2925 pcfg->num_peri, pcfg->num_events); 2926 2927 pm_runtime_irq_safe(&adev->dev); 2928 pm_runtime_use_autosuspend(&adev->dev); 2929 pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY); 2930 pm_runtime_mark_last_busy(&adev->dev); 2931 pm_runtime_put_autosuspend(&adev->dev); 2932 2933 return 0; 2934probe_err3: 2935 /* Idle the DMAC */ 2936 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels, 2937 chan.device_node) { 2938 2939 /* Remove the channel */ 2940 list_del(&pch->chan.device_node); 2941 2942 /* Flush the channel */ 2943 if (pch->thread) { 2944 pl330_terminate_all(&pch->chan); 2945 pl330_free_chan_resources(&pch->chan); 2946 } 2947 } 2948probe_err2: 2949 pl330_del(pl330); 2950 2951 return ret; 2952} 2953 2954static int pl330_remove(struct amba_device *adev) 2955{ 2956 struct pl330_dmac *pl330 = amba_get_drvdata(adev); 2957 struct dma_pl330_chan *pch, *_p; 2958 2959 pm_runtime_get_noresume(pl330->ddma.dev); 2960 2961 if (adev->dev.of_node) 2962 of_dma_controller_free(adev->dev.of_node); 2963 2964 dma_async_device_unregister(&pl330->ddma); 2965 2966 /* Idle the DMAC */ 2967 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels, 2968 chan.device_node) { 2969 2970 /* Remove the channel */ 2971 list_del(&pch->chan.device_node); 2972 2973 /* Flush the channel */ 2974 if (pch->thread) { 2975 pl330_terminate_all(&pch->chan); 2976 pl330_free_chan_resources(&pch->chan); 2977 } 2978 } 2979 2980 pl330_del(pl330); 2981 2982 return 0; 2983} 2984 2985static struct amba_id pl330_ids[] = { 2986 { 2987 .id = 0x00041330, 2988 .mask = 0x000fffff, 2989 }, 2990 { 0, 0 }, 2991}; 2992 2993MODULE_DEVICE_TABLE(amba, pl330_ids); 2994 2995static struct amba_driver pl330_driver = { 2996 .drv = { 2997 .owner = THIS_MODULE, 2998 .name = "dma-pl330", 2999 .pm = &pl330_pm, 3000 }, 3001 .id_table = pl330_ids, 3002 .probe = pl330_probe, 3003 .remove = pl330_remove, 3004}; 3005 3006module_amba_driver(pl330_driver); 3007 3008MODULE_AUTHOR("Jaswinder Singh <jassisinghbrar@gmail.com>"); 3009MODULE_DESCRIPTION("API Driver for PL330 DMAC"); 3010MODULE_LICENSE("GPL"); 3011