1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2009 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * The full GNU General Public License is included in this distribution in 15 * the file called "COPYING". 16 * 17 */ 18 19/* 20 * This driver supports an Intel I/OAT DMA engine, which does asynchronous 21 * copy operations. 22 */ 23 24#include <linux/init.h> 25#include <linux/module.h> 26#include <linux/slab.h> 27#include <linux/pci.h> 28#include <linux/interrupt.h> 29#include <linux/dmaengine.h> 30#include <linux/delay.h> 31#include <linux/dma-mapping.h> 32#include <linux/workqueue.h> 33#include <linux/prefetch.h> 34#include <linux/i7300_idle.h> 35#include "dma.h" 36#include "registers.h" 37#include "hw.h" 38 39#include "../dmaengine.h" 40 41int ioat_pending_level = 4; 42module_param(ioat_pending_level, int, 0644); 43MODULE_PARM_DESC(ioat_pending_level, 44 "high-water mark for pushing ioat descriptors (default: 4)"); 45 46/* internal functions */ 47static void ioat1_cleanup(struct ioat_dma_chan *ioat); 48static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat); 49 50/** 51 * ioat_dma_do_interrupt - handler used for single vector interrupt mode 52 * @irq: interrupt id 53 * @data: interrupt data 54 */ 55static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) 56{ 57 struct ioatdma_device *instance = data; 58 struct ioat_chan_common *chan; 59 unsigned long attnstatus; 60 int bit; 61 u8 intrctrl; 62 63 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); 64 65 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) 66 return IRQ_NONE; 67 68 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { 69 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); 70 return IRQ_NONE; 71 } 72 73 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); 74 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { 75 chan = ioat_chan_by_index(instance, bit); 76 if (test_bit(IOAT_RUN, &chan->state)) 77 tasklet_schedule(&chan->cleanup_task); 78 } 79 80 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); 81 return IRQ_HANDLED; 82} 83 84/** 85 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode 86 * @irq: interrupt id 87 * @data: interrupt data 88 */ 89static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) 90{ 91 struct ioat_chan_common *chan = data; 92 93 if (test_bit(IOAT_RUN, &chan->state)) 94 tasklet_schedule(&chan->cleanup_task); 95 96 return IRQ_HANDLED; 97} 98 99/* common channel initialization */ 100void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) 101{ 102 struct dma_device *dma = &device->common; 103 struct dma_chan *c = &chan->common; 104 unsigned long data = (unsigned long) c; 105 106 chan->device = device; 107 chan->reg_base = device->reg_base + (0x80 * (idx + 1)); 108 spin_lock_init(&chan->cleanup_lock); 109 chan->common.device = dma; 110 dma_cookie_init(&chan->common); 111 list_add_tail(&chan->common.device_node, &dma->channels); 112 device->idx[idx] = chan; 113 init_timer(&chan->timer); 114 chan->timer.function = device->timer_fn; 115 chan->timer.data = data; 116 tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); 117} 118 119/** 120 * ioat1_dma_enumerate_channels - find and initialize the device's channels 121 * @device: the device to be enumerated 122 */ 123static int ioat1_enumerate_channels(struct ioatdma_device *device) 124{ 125 u8 xfercap_scale; 126 u32 xfercap; 127 int i; 128 struct ioat_dma_chan *ioat; 129 struct device *dev = &device->pdev->dev; 130 struct dma_device *dma = &device->common; 131 132 INIT_LIST_HEAD(&dma->channels); 133 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); 134 dma->chancnt &= 0x1f; /* bits [4:0] valid */ 135 if (dma->chancnt > ARRAY_SIZE(device->idx)) { 136 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", 137 dma->chancnt, ARRAY_SIZE(device->idx)); 138 dma->chancnt = ARRAY_SIZE(device->idx); 139 } 140 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET); 141 xfercap_scale &= 0x1f; /* bits [4:0] valid */ 142 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); 143 dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap); 144 145#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL 146 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) 147 dma->chancnt--; 148#endif 149 for (i = 0; i < dma->chancnt; i++) { 150 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); 151 if (!ioat) 152 break; 153 154 ioat_init_channel(device, &ioat->base, i); 155 ioat->xfercap = xfercap; 156 spin_lock_init(&ioat->desc_lock); 157 INIT_LIST_HEAD(&ioat->free_desc); 158 INIT_LIST_HEAD(&ioat->used_desc); 159 } 160 dma->chancnt = i; 161 return i; 162} 163 164/** 165 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended 166 * descriptors to hw 167 * @chan: DMA channel handle 168 */ 169static inline void 170__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat) 171{ 172 void __iomem *reg_base = ioat->base.reg_base; 173 174 dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n", 175 __func__, ioat->pending); 176 ioat->pending = 0; 177 writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET); 178} 179 180static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan) 181{ 182 struct ioat_dma_chan *ioat = to_ioat_chan(chan); 183 184 if (ioat->pending > 0) { 185 spin_lock_bh(&ioat->desc_lock); 186 __ioat1_dma_memcpy_issue_pending(ioat); 187 spin_unlock_bh(&ioat->desc_lock); 188 } 189} 190 191/** 192 * ioat1_reset_channel - restart a channel 193 * @ioat: IOAT DMA channel handle 194 */ 195static void ioat1_reset_channel(struct ioat_dma_chan *ioat) 196{ 197 struct ioat_chan_common *chan = &ioat->base; 198 void __iomem *reg_base = chan->reg_base; 199 u32 chansts, chanerr; 200 201 dev_warn(to_dev(chan), "reset\n"); 202 chanerr = readl(reg_base + IOAT_CHANERR_OFFSET); 203 chansts = *chan->completion & IOAT_CHANSTS_STATUS; 204 if (chanerr) { 205 dev_err(to_dev(chan), 206 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n", 207 chan_num(chan), chansts, chanerr); 208 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET); 209 } 210 211 /* 212 * whack it upside the head with a reset 213 * and wait for things to settle out. 214 * force the pending count to a really big negative 215 * to make sure no one forces an issue_pending 216 * while we're waiting. 217 */ 218 219 ioat->pending = INT_MIN; 220 writeb(IOAT_CHANCMD_RESET, 221 reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); 222 set_bit(IOAT_RESET_PENDING, &chan->state); 223 mod_timer(&chan->timer, jiffies + RESET_DELAY); 224} 225 226static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx) 227{ 228 struct dma_chan *c = tx->chan; 229 struct ioat_dma_chan *ioat = to_ioat_chan(c); 230 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); 231 struct ioat_chan_common *chan = &ioat->base; 232 struct ioat_desc_sw *first; 233 struct ioat_desc_sw *chain_tail; 234 dma_cookie_t cookie; 235 236 spin_lock_bh(&ioat->desc_lock); 237 /* cookie incr and addition to used_list must be atomic */ 238 cookie = dma_cookie_assign(tx); 239 dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); 240 241 /* write address into NextDescriptor field of last desc in chain */ 242 first = to_ioat_desc(desc->tx_list.next); 243 chain_tail = to_ioat_desc(ioat->used_desc.prev); 244 /* make descriptor updates globally visible before chaining */ 245 wmb(); 246 chain_tail->hw->next = first->txd.phys; 247 list_splice_tail_init(&desc->tx_list, &ioat->used_desc); 248 dump_desc_dbg(ioat, chain_tail); 249 dump_desc_dbg(ioat, first); 250 251 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) 252 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 253 254 ioat->active += desc->hw->tx_cnt; 255 ioat->pending += desc->hw->tx_cnt; 256 if (ioat->pending >= ioat_pending_level) 257 __ioat1_dma_memcpy_issue_pending(ioat); 258 spin_unlock_bh(&ioat->desc_lock); 259 260 return cookie; 261} 262 263/** 264 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair 265 * @ioat: the channel supplying the memory pool for the descriptors 266 * @flags: allocation flags 267 */ 268static struct ioat_desc_sw * 269ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags) 270{ 271 struct ioat_dma_descriptor *desc; 272 struct ioat_desc_sw *desc_sw; 273 struct ioatdma_device *ioatdma_device; 274 dma_addr_t phys; 275 276 ioatdma_device = ioat->base.device; 277 desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys); 278 if (unlikely(!desc)) 279 return NULL; 280 281 desc_sw = kzalloc(sizeof(*desc_sw), flags); 282 if (unlikely(!desc_sw)) { 283 pci_pool_free(ioatdma_device->dma_pool, desc, phys); 284 return NULL; 285 } 286 287 memset(desc, 0, sizeof(*desc)); 288 289 INIT_LIST_HEAD(&desc_sw->tx_list); 290 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common); 291 desc_sw->txd.tx_submit = ioat1_tx_submit; 292 desc_sw->hw = desc; 293 desc_sw->txd.phys = phys; 294 set_desc_id(desc_sw, -1); 295 296 return desc_sw; 297} 298 299static int ioat_initial_desc_count = 256; 300module_param(ioat_initial_desc_count, int, 0644); 301MODULE_PARM_DESC(ioat_initial_desc_count, 302 "ioat1: initial descriptors per channel (default: 256)"); 303/** 304 * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors 305 * @chan: the channel to be filled out 306 */ 307static int ioat1_dma_alloc_chan_resources(struct dma_chan *c) 308{ 309 struct ioat_dma_chan *ioat = to_ioat_chan(c); 310 struct ioat_chan_common *chan = &ioat->base; 311 struct ioat_desc_sw *desc; 312 u32 chanerr; 313 int i; 314 LIST_HEAD(tmp_list); 315 316 /* have we already been set up? */ 317 if (!list_empty(&ioat->free_desc)) 318 return ioat->desccount; 319 320 /* Setup register to interrupt and write completion status on error */ 321 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); 322 323 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 324 if (chanerr) { 325 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr); 326 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); 327 } 328 329 /* Allocate descriptors */ 330 for (i = 0; i < ioat_initial_desc_count; i++) { 331 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL); 332 if (!desc) { 333 dev_err(to_dev(chan), "Only %d initial descriptors\n", i); 334 break; 335 } 336 set_desc_id(desc, i); 337 list_add_tail(&desc->node, &tmp_list); 338 } 339 spin_lock_bh(&ioat->desc_lock); 340 ioat->desccount = i; 341 list_splice(&tmp_list, &ioat->free_desc); 342 spin_unlock_bh(&ioat->desc_lock); 343 344 /* allocate a completion writeback area */ 345 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ 346 chan->completion = pci_pool_alloc(chan->device->completion_pool, 347 GFP_KERNEL, &chan->completion_dma); 348 memset(chan->completion, 0, sizeof(*chan->completion)); 349 writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, 350 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); 351 writel(((u64) chan->completion_dma) >> 32, 352 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); 353 354 set_bit(IOAT_RUN, &chan->state); 355 ioat1_dma_start_null_desc(ioat); /* give chain to dma device */ 356 dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", 357 __func__, ioat->desccount); 358 return ioat->desccount; 359} 360 361void ioat_stop(struct ioat_chan_common *chan) 362{ 363 struct ioatdma_device *device = chan->device; 364 struct pci_dev *pdev = device->pdev; 365 int chan_id = chan_num(chan); 366 struct msix_entry *msix; 367 368 /* 1/ stop irq from firing tasklets 369 * 2/ stop the tasklet from re-arming irqs 370 */ 371 clear_bit(IOAT_RUN, &chan->state); 372 373 /* flush inflight interrupts */ 374 switch (device->irq_mode) { 375 case IOAT_MSIX: 376 msix = &device->msix_entries[chan_id]; 377 synchronize_irq(msix->vector); 378 break; 379 case IOAT_MSI: 380 case IOAT_INTX: 381 synchronize_irq(pdev->irq); 382 break; 383 default: 384 break; 385 } 386 387 /* flush inflight timers */ 388 del_timer_sync(&chan->timer); 389 390 /* flush inflight tasklet runs */ 391 tasklet_kill(&chan->cleanup_task); 392 393 /* final cleanup now that everything is quiesced and can't re-arm */ 394 device->cleanup_fn((unsigned long) &chan->common); 395} 396 397/** 398 * ioat1_dma_free_chan_resources - release all the descriptors 399 * @chan: the channel to be cleaned 400 */ 401static void ioat1_dma_free_chan_resources(struct dma_chan *c) 402{ 403 struct ioat_dma_chan *ioat = to_ioat_chan(c); 404 struct ioat_chan_common *chan = &ioat->base; 405 struct ioatdma_device *ioatdma_device = chan->device; 406 struct ioat_desc_sw *desc, *_desc; 407 int in_use_descs = 0; 408 409 /* Before freeing channel resources first check 410 * if they have been previously allocated for this channel. 411 */ 412 if (ioat->desccount == 0) 413 return; 414 415 ioat_stop(chan); 416 417 /* Delay 100ms after reset to allow internal DMA logic to quiesce 418 * before removing DMA descriptor resources. 419 */ 420 writeb(IOAT_CHANCMD_RESET, 421 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); 422 mdelay(100); 423 424 spin_lock_bh(&ioat->desc_lock); 425 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) { 426 dev_dbg(to_dev(chan), "%s: freeing %d from used list\n", 427 __func__, desc_id(desc)); 428 dump_desc_dbg(ioat, desc); 429 in_use_descs++; 430 list_del(&desc->node); 431 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 432 desc->txd.phys); 433 kfree(desc); 434 } 435 list_for_each_entry_safe(desc, _desc, 436 &ioat->free_desc, node) { 437 list_del(&desc->node); 438 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 439 desc->txd.phys); 440 kfree(desc); 441 } 442 spin_unlock_bh(&ioat->desc_lock); 443 444 pci_pool_free(ioatdma_device->completion_pool, 445 chan->completion, 446 chan->completion_dma); 447 448 /* one is ok since we left it on there on purpose */ 449 if (in_use_descs > 1) 450 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", 451 in_use_descs - 1); 452 453 chan->last_completion = 0; 454 chan->completion_dma = 0; 455 ioat->pending = 0; 456 ioat->desccount = 0; 457} 458 459/** 460 * ioat1_dma_get_next_descriptor - return the next available descriptor 461 * @ioat: IOAT DMA channel handle 462 * 463 * Gets the next descriptor from the chain, and must be called with the 464 * channel's desc_lock held. Allocates more descriptors if the channel 465 * has run out. 466 */ 467static struct ioat_desc_sw * 468ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat) 469{ 470 struct ioat_desc_sw *new; 471 472 if (!list_empty(&ioat->free_desc)) { 473 new = to_ioat_desc(ioat->free_desc.next); 474 list_del(&new->node); 475 } else { 476 /* try to get another desc */ 477 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC); 478 if (!new) { 479 dev_err(to_dev(&ioat->base), "alloc failed\n"); 480 return NULL; 481 } 482 } 483 dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n", 484 __func__, desc_id(new)); 485 prefetch(new->hw); 486 return new; 487} 488 489static struct dma_async_tx_descriptor * 490ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, 491 dma_addr_t dma_src, size_t len, unsigned long flags) 492{ 493 struct ioat_dma_chan *ioat = to_ioat_chan(c); 494 struct ioat_desc_sw *desc; 495 size_t copy; 496 LIST_HEAD(chain); 497 dma_addr_t src = dma_src; 498 dma_addr_t dest = dma_dest; 499 size_t total_len = len; 500 struct ioat_dma_descriptor *hw = NULL; 501 int tx_cnt = 0; 502 503 spin_lock_bh(&ioat->desc_lock); 504 desc = ioat1_dma_get_next_descriptor(ioat); 505 do { 506 if (!desc) 507 break; 508 509 tx_cnt++; 510 copy = min_t(size_t, len, ioat->xfercap); 511 512 hw = desc->hw; 513 hw->size = copy; 514 hw->ctl = 0; 515 hw->src_addr = src; 516 hw->dst_addr = dest; 517 518 list_add_tail(&desc->node, &chain); 519 520 len -= copy; 521 dest += copy; 522 src += copy; 523 if (len) { 524 struct ioat_desc_sw *next; 525 526 async_tx_ack(&desc->txd); 527 next = ioat1_dma_get_next_descriptor(ioat); 528 hw->next = next ? next->txd.phys : 0; 529 dump_desc_dbg(ioat, desc); 530 desc = next; 531 } else 532 hw->next = 0; 533 } while (len); 534 535 if (!desc) { 536 struct ioat_chan_common *chan = &ioat->base; 537 538 dev_err(to_dev(chan), 539 "chan%d - get_next_desc failed\n", chan_num(chan)); 540 list_splice(&chain, &ioat->free_desc); 541 spin_unlock_bh(&ioat->desc_lock); 542 return NULL; 543 } 544 spin_unlock_bh(&ioat->desc_lock); 545 546 desc->txd.flags = flags; 547 desc->len = total_len; 548 list_splice(&chain, &desc->tx_list); 549 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 550 hw->ctl_f.compl_write = 1; 551 hw->tx_cnt = tx_cnt; 552 dump_desc_dbg(ioat, desc); 553 554 return &desc->txd; 555} 556 557static void ioat1_cleanup_event(unsigned long data) 558{ 559 struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); 560 struct ioat_chan_common *chan = &ioat->base; 561 562 ioat1_cleanup(ioat); 563 if (!test_bit(IOAT_RUN, &chan->state)) 564 return; 565 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); 566} 567 568dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) 569{ 570 dma_addr_t phys_complete; 571 u64 completion; 572 573 completion = *chan->completion; 574 phys_complete = ioat_chansts_to_addr(completion); 575 576 dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, 577 (unsigned long long) phys_complete); 578 579 if (is_ioat_halted(completion)) { 580 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 581 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n", 582 chanerr); 583 584 /* TODO do something to salvage the situation */ 585 } 586 587 return phys_complete; 588} 589 590bool ioat_cleanup_preamble(struct ioat_chan_common *chan, 591 dma_addr_t *phys_complete) 592{ 593 *phys_complete = ioat_get_current_completion(chan); 594 if (*phys_complete == chan->last_completion) 595 return false; 596 clear_bit(IOAT_COMPLETION_ACK, &chan->state); 597 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 598 599 return true; 600} 601 602static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete) 603{ 604 struct ioat_chan_common *chan = &ioat->base; 605 struct list_head *_desc, *n; 606 struct dma_async_tx_descriptor *tx; 607 608 dev_dbg(to_dev(chan), "%s: phys_complete: %llx\n", 609 __func__, (unsigned long long) phys_complete); 610 list_for_each_safe(_desc, n, &ioat->used_desc) { 611 struct ioat_desc_sw *desc; 612 613 prefetch(n); 614 desc = list_entry(_desc, typeof(*desc), node); 615 tx = &desc->txd; 616 /* 617 * Incoming DMA requests may use multiple descriptors, 618 * due to exceeding xfercap, perhaps. If so, only the 619 * last one will have a cookie, and require unmapping. 620 */ 621 dump_desc_dbg(ioat, desc); 622 if (tx->cookie) { 623 dma_cookie_complete(tx); 624 dma_descriptor_unmap(tx); 625 ioat->active -= desc->hw->tx_cnt; 626 if (tx->callback) { 627 tx->callback(tx->callback_param); 628 tx->callback = NULL; 629 } 630 } 631 632 if (tx->phys != phys_complete) { 633 /* 634 * a completed entry, but not the last, so clean 635 * up if the client is done with the descriptor 636 */ 637 if (async_tx_test_ack(tx)) 638 list_move_tail(&desc->node, &ioat->free_desc); 639 } else { 640 /* 641 * last used desc. Do not remove, so we can 642 * append from it. 643 */ 644 645 /* if nothing else is pending, cancel the 646 * completion timeout 647 */ 648 if (n == &ioat->used_desc) { 649 dev_dbg(to_dev(chan), 650 "%s cancel completion timeout\n", 651 __func__); 652 clear_bit(IOAT_COMPLETION_PENDING, &chan->state); 653 } 654 655 /* TODO check status bits? */ 656 break; 657 } 658 } 659 660 chan->last_completion = phys_complete; 661} 662 663/** 664 * ioat1_cleanup - cleanup up finished descriptors 665 * @chan: ioat channel to be cleaned up 666 * 667 * To prevent lock contention we defer cleanup when the locks are 668 * contended with a terminal timeout that forces cleanup and catches 669 * completion notification errors. 670 */ 671static void ioat1_cleanup(struct ioat_dma_chan *ioat) 672{ 673 struct ioat_chan_common *chan = &ioat->base; 674 dma_addr_t phys_complete; 675 676 prefetch(chan->completion); 677 678 if (!spin_trylock_bh(&chan->cleanup_lock)) 679 return; 680 681 if (!ioat_cleanup_preamble(chan, &phys_complete)) { 682 spin_unlock_bh(&chan->cleanup_lock); 683 return; 684 } 685 686 if (!spin_trylock_bh(&ioat->desc_lock)) { 687 spin_unlock_bh(&chan->cleanup_lock); 688 return; 689 } 690 691 __cleanup(ioat, phys_complete); 692 693 spin_unlock_bh(&ioat->desc_lock); 694 spin_unlock_bh(&chan->cleanup_lock); 695} 696 697static void ioat1_timer_event(unsigned long data) 698{ 699 struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); 700 struct ioat_chan_common *chan = &ioat->base; 701 702 dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state); 703 704 spin_lock_bh(&chan->cleanup_lock); 705 if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) { 706 struct ioat_desc_sw *desc; 707 708 spin_lock_bh(&ioat->desc_lock); 709 710 /* restart active descriptors */ 711 desc = to_ioat_desc(ioat->used_desc.prev); 712 ioat_set_chainaddr(ioat, desc->txd.phys); 713 ioat_start(chan); 714 715 ioat->pending = 0; 716 set_bit(IOAT_COMPLETION_PENDING, &chan->state); 717 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 718 spin_unlock_bh(&ioat->desc_lock); 719 } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { 720 dma_addr_t phys_complete; 721 722 spin_lock_bh(&ioat->desc_lock); 723 /* if we haven't made progress and we have already 724 * acknowledged a pending completion once, then be more 725 * forceful with a restart 726 */ 727 if (ioat_cleanup_preamble(chan, &phys_complete)) 728 __cleanup(ioat, phys_complete); 729 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) 730 ioat1_reset_channel(ioat); 731 else { 732 u64 status = ioat_chansts(chan); 733 734 /* manually update the last completion address */ 735 if (ioat_chansts_to_addr(status) != 0) 736 *chan->completion = status; 737 738 set_bit(IOAT_COMPLETION_ACK, &chan->state); 739 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 740 } 741 spin_unlock_bh(&ioat->desc_lock); 742 } 743 spin_unlock_bh(&chan->cleanup_lock); 744} 745 746enum dma_status 747ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, 748 struct dma_tx_state *txstate) 749{ 750 struct ioat_chan_common *chan = to_chan_common(c); 751 struct ioatdma_device *device = chan->device; 752 enum dma_status ret; 753 754 ret = dma_cookie_status(c, cookie, txstate); 755 if (ret == DMA_COMPLETE) 756 return ret; 757 758 device->cleanup_fn((unsigned long) c); 759 760 return dma_cookie_status(c, cookie, txstate); 761} 762 763static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat) 764{ 765 struct ioat_chan_common *chan = &ioat->base; 766 struct ioat_desc_sw *desc; 767 struct ioat_dma_descriptor *hw; 768 769 spin_lock_bh(&ioat->desc_lock); 770 771 desc = ioat1_dma_get_next_descriptor(ioat); 772 773 if (!desc) { 774 dev_err(to_dev(chan), 775 "Unable to start null desc - get next desc failed\n"); 776 spin_unlock_bh(&ioat->desc_lock); 777 return; 778 } 779 780 hw = desc->hw; 781 hw->ctl = 0; 782 hw->ctl_f.null = 1; 783 hw->ctl_f.int_en = 1; 784 hw->ctl_f.compl_write = 1; 785 /* set size to non-zero value (channel returns error when size is 0) */ 786 hw->size = NULL_DESC_BUFFER_SIZE; 787 hw->src_addr = 0; 788 hw->dst_addr = 0; 789 async_tx_ack(&desc->txd); 790 hw->next = 0; 791 list_add_tail(&desc->node, &ioat->used_desc); 792 dump_desc_dbg(ioat, desc); 793 794 ioat_set_chainaddr(ioat, desc->txd.phys); 795 ioat_start(chan); 796 spin_unlock_bh(&ioat->desc_lock); 797} 798 799/* 800 * Perform a IOAT transaction to verify the HW works. 801 */ 802#define IOAT_TEST_SIZE 2000 803 804static void ioat_dma_test_callback(void *dma_async_param) 805{ 806 struct completion *cmp = dma_async_param; 807 808 complete(cmp); 809} 810 811/** 812 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. 813 * @device: device to be tested 814 */ 815int ioat_dma_self_test(struct ioatdma_device *device) 816{ 817 int i; 818 u8 *src; 819 u8 *dest; 820 struct dma_device *dma = &device->common; 821 struct device *dev = &device->pdev->dev; 822 struct dma_chan *dma_chan; 823 struct dma_async_tx_descriptor *tx; 824 dma_addr_t dma_dest, dma_src; 825 dma_cookie_t cookie; 826 int err = 0; 827 struct completion cmp; 828 unsigned long tmo; 829 unsigned long flags; 830 831 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); 832 if (!src) 833 return -ENOMEM; 834 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); 835 if (!dest) { 836 kfree(src); 837 return -ENOMEM; 838 } 839 840 /* Fill in src buffer */ 841 for (i = 0; i < IOAT_TEST_SIZE; i++) 842 src[i] = (u8)i; 843 844 /* Start copy, using first DMA channel */ 845 dma_chan = container_of(dma->channels.next, struct dma_chan, 846 device_node); 847 if (dma->device_alloc_chan_resources(dma_chan) < 1) { 848 dev_err(dev, "selftest cannot allocate chan resource\n"); 849 err = -ENODEV; 850 goto out; 851 } 852 853 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 854 if (dma_mapping_error(dev, dma_src)) { 855 dev_err(dev, "mapping src buffer failed\n"); 856 goto free_resources; 857 } 858 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 859 if (dma_mapping_error(dev, dma_dest)) { 860 dev_err(dev, "mapping dest buffer failed\n"); 861 goto unmap_src; 862 } 863 flags = DMA_PREP_INTERRUPT; 864 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, 865 IOAT_TEST_SIZE, flags); 866 if (!tx) { 867 dev_err(dev, "Self-test prep failed, disabling\n"); 868 err = -ENODEV; 869 goto unmap_dma; 870 } 871 872 async_tx_ack(tx); 873 init_completion(&cmp); 874 tx->callback = ioat_dma_test_callback; 875 tx->callback_param = &cmp; 876 cookie = tx->tx_submit(tx); 877 if (cookie < 0) { 878 dev_err(dev, "Self-test setup failed, disabling\n"); 879 err = -ENODEV; 880 goto unmap_dma; 881 } 882 dma->device_issue_pending(dma_chan); 883 884 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); 885 886 if (tmo == 0 || 887 dma->device_tx_status(dma_chan, cookie, NULL) 888 != DMA_COMPLETE) { 889 dev_err(dev, "Self-test copy timed out, disabling\n"); 890 err = -ENODEV; 891 goto unmap_dma; 892 } 893 if (memcmp(src, dest, IOAT_TEST_SIZE)) { 894 dev_err(dev, "Self-test copy failed compare, disabling\n"); 895 err = -ENODEV; 896 goto free_resources; 897 } 898 899unmap_dma: 900 dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 901unmap_src: 902 dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 903free_resources: 904 dma->device_free_chan_resources(dma_chan); 905out: 906 kfree(src); 907 kfree(dest); 908 return err; 909} 910 911static char ioat_interrupt_style[32] = "msix"; 912module_param_string(ioat_interrupt_style, ioat_interrupt_style, 913 sizeof(ioat_interrupt_style), 0644); 914MODULE_PARM_DESC(ioat_interrupt_style, 915 "set ioat interrupt style: msix (default), msi, intx"); 916 917/** 918 * ioat_dma_setup_interrupts - setup interrupt handler 919 * @device: ioat device 920 */ 921int ioat_dma_setup_interrupts(struct ioatdma_device *device) 922{ 923 struct ioat_chan_common *chan; 924 struct pci_dev *pdev = device->pdev; 925 struct device *dev = &pdev->dev; 926 struct msix_entry *msix; 927 int i, j, msixcnt; 928 int err = -EINVAL; 929 u8 intrctrl = 0; 930 931 if (!strcmp(ioat_interrupt_style, "msix")) 932 goto msix; 933 if (!strcmp(ioat_interrupt_style, "msi")) 934 goto msi; 935 if (!strcmp(ioat_interrupt_style, "intx")) 936 goto intx; 937 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style); 938 goto err_no_irq; 939 940msix: 941 /* The number of MSI-X vectors should equal the number of channels */ 942 msixcnt = device->common.chancnt; 943 for (i = 0; i < msixcnt; i++) 944 device->msix_entries[i].entry = i; 945 946 err = pci_enable_msix_exact(pdev, device->msix_entries, msixcnt); 947 if (err) 948 goto msi; 949 950 for (i = 0; i < msixcnt; i++) { 951 msix = &device->msix_entries[i]; 952 chan = ioat_chan_by_index(device, i); 953 err = devm_request_irq(dev, msix->vector, 954 ioat_dma_do_interrupt_msix, 0, 955 "ioat-msix", chan); 956 if (err) { 957 for (j = 0; j < i; j++) { 958 msix = &device->msix_entries[j]; 959 chan = ioat_chan_by_index(device, j); 960 devm_free_irq(dev, msix->vector, chan); 961 } 962 goto msi; 963 } 964 } 965 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; 966 device->irq_mode = IOAT_MSIX; 967 goto done; 968 969msi: 970 err = pci_enable_msi(pdev); 971 if (err) 972 goto intx; 973 974 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, 975 "ioat-msi", device); 976 if (err) { 977 pci_disable_msi(pdev); 978 goto intx; 979 } 980 device->irq_mode = IOAT_MSI; 981 goto done; 982 983intx: 984 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 985 IRQF_SHARED, "ioat-intx", device); 986 if (err) 987 goto err_no_irq; 988 989 device->irq_mode = IOAT_INTX; 990done: 991 if (device->intr_quirk) 992 device->intr_quirk(device); 993 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; 994 writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET); 995 return 0; 996 997err_no_irq: 998 /* Disable all interrupt generation */ 999 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); 1000 device->irq_mode = IOAT_NOIRQ; 1001 dev_err(dev, "no usable interrupts\n"); 1002 return err; 1003} 1004EXPORT_SYMBOL(ioat_dma_setup_interrupts); 1005 1006static void ioat_disable_interrupts(struct ioatdma_device *device) 1007{ 1008 /* Disable all interrupt generation */ 1009 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); 1010} 1011 1012int ioat_probe(struct ioatdma_device *device) 1013{ 1014 int err = -ENODEV; 1015 struct dma_device *dma = &device->common; 1016 struct pci_dev *pdev = device->pdev; 1017 struct device *dev = &pdev->dev; 1018 1019 /* DMA coherent memory pool for DMA descriptor allocations */ 1020 device->dma_pool = pci_pool_create("dma_desc_pool", pdev, 1021 sizeof(struct ioat_dma_descriptor), 1022 64, 0); 1023 if (!device->dma_pool) { 1024 err = -ENOMEM; 1025 goto err_dma_pool; 1026 } 1027 1028 device->completion_pool = pci_pool_create("completion_pool", pdev, 1029 sizeof(u64), SMP_CACHE_BYTES, 1030 SMP_CACHE_BYTES); 1031 1032 if (!device->completion_pool) { 1033 err = -ENOMEM; 1034 goto err_completion_pool; 1035 } 1036 1037 device->enumerate_channels(device); 1038 1039 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 1040 dma->dev = &pdev->dev; 1041 1042 if (!dma->chancnt) { 1043 dev_err(dev, "channel enumeration error\n"); 1044 goto err_setup_interrupts; 1045 } 1046 1047 err = ioat_dma_setup_interrupts(device); 1048 if (err) 1049 goto err_setup_interrupts; 1050 1051 err = device->self_test(device); 1052 if (err) 1053 goto err_self_test; 1054 1055 return 0; 1056 1057err_self_test: 1058 ioat_disable_interrupts(device); 1059err_setup_interrupts: 1060 pci_pool_destroy(device->completion_pool); 1061err_completion_pool: 1062 pci_pool_destroy(device->dma_pool); 1063err_dma_pool: 1064 return err; 1065} 1066 1067int ioat_register(struct ioatdma_device *device) 1068{ 1069 int err = dma_async_device_register(&device->common); 1070 1071 if (err) { 1072 ioat_disable_interrupts(device); 1073 pci_pool_destroy(device->completion_pool); 1074 pci_pool_destroy(device->dma_pool); 1075 } 1076 1077 return err; 1078} 1079 1080/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */ 1081static void ioat1_intr_quirk(struct ioatdma_device *device) 1082{ 1083 struct pci_dev *pdev = device->pdev; 1084 u32 dmactrl; 1085 1086 pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl); 1087 if (pdev->msi_enabled) 1088 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN; 1089 else 1090 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN; 1091 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl); 1092} 1093 1094static ssize_t ring_size_show(struct dma_chan *c, char *page) 1095{ 1096 struct ioat_dma_chan *ioat = to_ioat_chan(c); 1097 1098 return sprintf(page, "%d\n", ioat->desccount); 1099} 1100static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); 1101 1102static ssize_t ring_active_show(struct dma_chan *c, char *page) 1103{ 1104 struct ioat_dma_chan *ioat = to_ioat_chan(c); 1105 1106 return sprintf(page, "%d\n", ioat->active); 1107} 1108static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); 1109 1110static ssize_t cap_show(struct dma_chan *c, char *page) 1111{ 1112 struct dma_device *dma = c->device; 1113 1114 return sprintf(page, "copy%s%s%s%s%s\n", 1115 dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", 1116 dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "", 1117 dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "", 1118 dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "", 1119 dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); 1120 1121} 1122struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); 1123 1124static ssize_t version_show(struct dma_chan *c, char *page) 1125{ 1126 struct dma_device *dma = c->device; 1127 struct ioatdma_device *device = to_ioatdma_device(dma); 1128 1129 return sprintf(page, "%d.%d\n", 1130 device->version >> 4, device->version & 0xf); 1131} 1132struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); 1133 1134static struct attribute *ioat1_attrs[] = { 1135 &ring_size_attr.attr, 1136 &ring_active_attr.attr, 1137 &ioat_cap_attr.attr, 1138 &ioat_version_attr.attr, 1139 NULL, 1140}; 1141 1142static ssize_t 1143ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) 1144{ 1145 struct ioat_sysfs_entry *entry; 1146 struct ioat_chan_common *chan; 1147 1148 entry = container_of(attr, struct ioat_sysfs_entry, attr); 1149 chan = container_of(kobj, struct ioat_chan_common, kobj); 1150 1151 if (!entry->show) 1152 return -EIO; 1153 return entry->show(&chan->common, page); 1154} 1155 1156const struct sysfs_ops ioat_sysfs_ops = { 1157 .show = ioat_attr_show, 1158}; 1159 1160static struct kobj_type ioat1_ktype = { 1161 .sysfs_ops = &ioat_sysfs_ops, 1162 .default_attrs = ioat1_attrs, 1163}; 1164 1165void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type) 1166{ 1167 struct dma_device *dma = &device->common; 1168 struct dma_chan *c; 1169 1170 list_for_each_entry(c, &dma->channels, device_node) { 1171 struct ioat_chan_common *chan = to_chan_common(c); 1172 struct kobject *parent = &c->dev->device.kobj; 1173 int err; 1174 1175 err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); 1176 if (err) { 1177 dev_warn(to_dev(chan), 1178 "sysfs init error (%d), continuing...\n", err); 1179 kobject_put(&chan->kobj); 1180 set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state); 1181 } 1182 } 1183} 1184 1185void ioat_kobject_del(struct ioatdma_device *device) 1186{ 1187 struct dma_device *dma = &device->common; 1188 struct dma_chan *c; 1189 1190 list_for_each_entry(c, &dma->channels, device_node) { 1191 struct ioat_chan_common *chan = to_chan_common(c); 1192 1193 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) { 1194 kobject_del(&chan->kobj); 1195 kobject_put(&chan->kobj); 1196 } 1197 } 1198} 1199 1200int ioat1_dma_probe(struct ioatdma_device *device, int dca) 1201{ 1202 struct pci_dev *pdev = device->pdev; 1203 struct dma_device *dma; 1204 int err; 1205 1206 device->intr_quirk = ioat1_intr_quirk; 1207 device->enumerate_channels = ioat1_enumerate_channels; 1208 device->self_test = ioat_dma_self_test; 1209 device->timer_fn = ioat1_timer_event; 1210 device->cleanup_fn = ioat1_cleanup_event; 1211 dma = &device->common; 1212 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; 1213 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; 1214 dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources; 1215 dma->device_free_chan_resources = ioat1_dma_free_chan_resources; 1216 dma->device_tx_status = ioat_dma_tx_status; 1217 1218 err = ioat_probe(device); 1219 if (err) 1220 return err; 1221 err = ioat_register(device); 1222 if (err) 1223 return err; 1224 ioat_kobject_add(device, &ioat1_ktype); 1225 1226 if (dca) 1227 device->dca = ioat_dca_init(pdev, device->reg_base); 1228 1229 return err; 1230} 1231 1232void ioat_dma_remove(struct ioatdma_device *device) 1233{ 1234 struct dma_device *dma = &device->common; 1235 1236 ioat_disable_interrupts(device); 1237 1238 ioat_kobject_del(device); 1239 1240 dma_async_device_unregister(dma); 1241 1242 pci_pool_destroy(device->dma_pool); 1243 pci_pool_destroy(device->completion_pool); 1244 1245 INIT_LIST_HEAD(&dma->channels); 1246} 1247