H A D | iop-adma.c | 46 * Caller must hold &iop_chan->lock while calling this function 62 struct iop_adma_chan *iop_chan, dma_cookie_t cookie) iop_adma_run_tx_complete_actions() 90 struct iop_adma_chan *iop_chan) iop_adma_clean_slot() 101 if (desc->chain_node.next == &iop_chan->chain) iop_adma_clean_slot() 104 dev_dbg(iop_chan->device->common.dev, iop_adma_clean_slot() 114 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) __iop_adma_slot_cleanup() argument 118 u32 current_desc = iop_chan_get_current_descriptor(iop_chan); __iop_adma_slot_cleanup() 119 int busy = iop_chan_is_busy(iop_chan); __iop_adma_slot_cleanup() 122 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); __iop_adma_slot_cleanup() 126 list_for_each_entry_safe(iter, _iter, &iop_chan->chain, __iop_adma_slot_cleanup() 183 &iop_chan->chain, chain_node) { __iop_adma_slot_cleanup() 201 &iop_chan->chain, chain_node) { __iop_adma_slot_cleanup() 203 grp_iter, iop_chan, cookie); __iop_adma_slot_cleanup() 207 iop_chan); __iop_adma_slot_cleanup() 231 iter, iop_chan, cookie); __iop_adma_slot_cleanup() 233 if (iop_adma_clean_slot(iter, iop_chan)) __iop_adma_slot_cleanup() 238 iop_chan->common.completed_cookie = cookie; __iop_adma_slot_cleanup() 244 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) iop_adma_slot_cleanup() argument 246 spin_lock_bh(&iop_chan->lock); iop_adma_slot_cleanup() 247 __iop_adma_slot_cleanup(iop_chan); iop_adma_slot_cleanup() 248 spin_unlock_bh(&iop_chan->lock); iop_adma_slot_cleanup() 253 struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data; iop_adma_tasklet() local 260 spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); iop_adma_tasklet() 261 __iop_adma_slot_cleanup(iop_chan); iop_adma_tasklet() 262 spin_unlock(&iop_chan->lock); iop_adma_tasklet() 266 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, iop_adma_alloc_slots() argument 280 iter = iop_chan->last_used; iop_adma_alloc_slots() 282 iter = list_entry(&iop_chan->all_slots, iop_adma_alloc_slots() 287 iter, _iter, &iop_chan->all_slots, slot_node) { iop_adma_alloc_slots() 317 dev_dbg(iop_chan->device->common.dev, iop_adma_alloc_slots() 344 iop_chan->last_used = last_used; iop_adma_alloc_slots() 354 __iop_adma_slot_cleanup(iop_chan); iop_adma_alloc_slots() 359 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) iop_adma_check_threshold() argument 361 dev_dbg(iop_chan->device->common.dev, "pending: %d\n", iop_adma_check_threshold() 362 iop_chan->pending); iop_adma_check_threshold() 364 if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { iop_adma_check_threshold() 365 iop_chan->pending = 0; iop_adma_check_threshold() 366 iop_chan_append(iop_chan); iop_adma_check_threshold() 374 struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); iop_adma_tx_submit() local 385 spin_lock_bh(&iop_chan->lock); iop_adma_tx_submit() 388 old_chain_tail = list_entry(iop_chan->chain.prev, iop_adma_tx_submit() 406 iop_chan->pending += slot_cnt; iop_adma_tx_submit() 407 iop_adma_check_threshold(iop_chan); iop_adma_tx_submit() 408 spin_unlock_bh(&iop_chan->lock); iop_adma_tx_submit() 410 dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", iop_adma_tx_submit() 416 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); 417 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); 424 * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To 433 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_alloc_chan_resources() local 435 int init = iop_chan->slots_allocated ? 0 : 1; iop_adma_alloc_chan_resources() 437 dev_get_platdata(&iop_chan->device->pdev->dev); iop_adma_alloc_chan_resources() 442 idx = iop_chan->slots_allocated; iop_adma_alloc_chan_resources() 452 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; iop_adma_alloc_chan_resources() 460 hw_desc = (char *) iop_chan->device->dma_desc_pool; iop_adma_alloc_chan_resources() 465 spin_lock_bh(&iop_chan->lock); iop_adma_alloc_chan_resources() 466 iop_chan->slots_allocated++; iop_adma_alloc_chan_resources() 467 list_add_tail(&slot->slot_node, &iop_chan->all_slots); iop_adma_alloc_chan_resources() 468 spin_unlock_bh(&iop_chan->lock); iop_adma_alloc_chan_resources() 469 } while (iop_chan->slots_allocated < num_descs_in_pool); iop_adma_alloc_chan_resources() 471 if (idx && !iop_chan->last_used) iop_adma_alloc_chan_resources() 472 iop_chan->last_used = list_entry(iop_chan->all_slots.next, iop_adma_alloc_chan_resources() 476 dev_dbg(iop_chan->device->common.dev, iop_adma_alloc_chan_resources() 478 iop_chan->slots_allocated, iop_chan->last_used); iop_adma_alloc_chan_resources() 483 iop_chan->device->common.cap_mask)) iop_adma_alloc_chan_resources() 484 iop_chan_start_null_memcpy(iop_chan); iop_adma_alloc_chan_resources() 486 iop_chan->device->common.cap_mask)) iop_adma_alloc_chan_resources() 487 iop_chan_start_null_xor(iop_chan); iop_adma_alloc_chan_resources() 498 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_prep_dma_interrupt() local 502 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); iop_adma_prep_dma_interrupt() 504 spin_lock_bh(&iop_chan->lock); iop_adma_prep_dma_interrupt() 505 slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); iop_adma_prep_dma_interrupt() 506 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_adma_prep_dma_interrupt() 509 iop_desc_init_interrupt(grp_start, iop_chan); iop_adma_prep_dma_interrupt() 512 spin_unlock_bh(&iop_chan->lock); iop_adma_prep_dma_interrupt() 521 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_prep_dma_memcpy() local 529 dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", iop_adma_prep_dma_memcpy() 532 spin_lock_bh(&iop_chan->lock); iop_adma_prep_dma_memcpy() 534 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_adma_prep_dma_memcpy() 538 iop_desc_set_byte_count(grp_start, iop_chan, len); iop_adma_prep_dma_memcpy() 539 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); iop_adma_prep_dma_memcpy() 543 spin_unlock_bh(&iop_chan->lock); iop_adma_prep_dma_memcpy() 553 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_prep_dma_xor() local 561 dev_dbg(iop_chan->device->common.dev, iop_adma_prep_dma_xor() 565 spin_lock_bh(&iop_chan->lock); iop_adma_prep_dma_xor() 567 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_adma_prep_dma_xor() 571 iop_desc_set_byte_count(grp_start, iop_chan, len); iop_adma_prep_dma_xor() 572 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); iop_adma_prep_dma_xor() 578 spin_unlock_bh(&iop_chan->lock); iop_adma_prep_dma_xor() 588 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_prep_dma_xor_val() local 595 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", iop_adma_prep_dma_xor_val() 598 spin_lock_bh(&iop_chan->lock); iop_adma_prep_dma_xor_val() 600 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_adma_prep_dma_xor_val() 613 spin_unlock_bh(&iop_chan->lock); iop_adma_prep_dma_xor_val() 623 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_prep_dma_pq() local 632 dev_dbg(iop_chan->device->common.dev, iop_adma_prep_dma_pq() 643 spin_lock_bh(&iop_chan->lock); iop_adma_prep_dma_pq() 645 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_adma_prep_dma_pq() 650 iop_desc_set_byte_count(g, iop_chan, len); iop_adma_prep_dma_pq() 677 spin_unlock_bh(&iop_chan->lock); iop_adma_prep_dma_pq() 688 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_prep_dma_pq_val() local 696 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", iop_adma_prep_dma_pq_val() 699 spin_lock_bh(&iop_chan->lock); iop_adma_prep_dma_pq_val() 701 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_adma_prep_dma_pq_val() 721 spin_unlock_bh(&iop_chan->lock); iop_adma_prep_dma_pq_val() 728 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_free_chan_resources() local 732 iop_adma_slot_cleanup(iop_chan); iop_adma_free_chan_resources() 734 spin_lock_bh(&iop_chan->lock); iop_adma_free_chan_resources() 735 list_for_each_entry_safe(iter, _iter, &iop_chan->chain, iop_adma_free_chan_resources() 741 iter, _iter, &iop_chan->all_slots, slot_node) { iop_adma_free_chan_resources() 744 iop_chan->slots_allocated--; iop_adma_free_chan_resources() 746 iop_chan->last_used = NULL; iop_adma_free_chan_resources() 748 dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", iop_adma_free_chan_resources() 749 __func__, iop_chan->slots_allocated); iop_adma_free_chan_resources() 750 spin_unlock_bh(&iop_chan->lock); iop_adma_free_chan_resources() 768 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_status() local 775 iop_adma_slot_cleanup(iop_chan); iop_adma_status() 830 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); iop_adma_issue_pending() local 832 if (iop_chan->pending) { iop_adma_issue_pending() 833 iop_chan->pending = 0; iop_adma_issue_pending() 834 iop_chan_append(iop_chan); iop_adma_issue_pending() 852 struct iop_adma_chan *iop_chan; iop_adma_memcpy_self_test() local 898 iop_chan = to_iop_adma_chan(dma_chan); iop_adma_memcpy_self_test() 899 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, iop_adma_memcpy_self_test() 933 struct iop_adma_chan *iop_chan; iop_adma_xor_val_self_test() local 998 iop_chan = to_iop_adma_chan(dma_chan); iop_adma_xor_val_self_test() 999 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, iop_adma_xor_val_self_test() 1010 dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, iop_adma_xor_val_self_test() 1258 struct iop_adma_chan *iop_chan; iop_adma_remove() local 1268 iop_chan = to_iop_adma_chan(chan); iop_adma_remove() 1270 kfree(iop_chan); iop_adma_remove() 1282 struct iop_adma_chan *iop_chan; iop_adma_probe() local 1353 iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); iop_adma_probe() 1354 if (!iop_chan) { iop_adma_probe() 1358 iop_chan->device = adev; iop_adma_probe() 1360 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, iop_adma_probe() 1362 if (!iop_chan->mmr_base) { iop_adma_probe() 1366 tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long) iop_adma_probe() 1367 iop_chan); iop_adma_probe() 1370 iop_adma_device_clear_err_status(iop_chan); iop_adma_probe() 1382 handler[i], 0, pdev->name, iop_chan); iop_adma_probe() 1388 spin_lock_init(&iop_chan->lock); iop_adma_probe() 1389 INIT_LIST_HEAD(&iop_chan->chain); iop_adma_probe() 1390 INIT_LIST_HEAD(&iop_chan->all_slots); iop_adma_probe() 1391 iop_chan->common.device = dma_dev; iop_adma_probe() 1392 dma_cookie_init(&iop_chan->common); iop_adma_probe() 1393 list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); iop_adma_probe() 1436 kfree(iop_chan); iop_adma_probe() 1446 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) iop_chan_start_null_memcpy() argument 1452 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); iop_chan_start_null_memcpy() 1454 spin_lock_bh(&iop_chan->lock); iop_chan_start_null_memcpy() 1456 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_chan_start_null_memcpy() 1460 list_splice_init(&sw_desc->tx_list, &iop_chan->chain); iop_chan_start_null_memcpy() 1463 iop_desc_set_byte_count(grp_start, iop_chan, 0); iop_chan_start_null_memcpy() 1464 iop_desc_set_dest_addr(grp_start, iop_chan, 0); iop_chan_start_null_memcpy() 1472 iop_chan->common.completed_cookie = cookie - 1; iop_chan_start_null_memcpy() 1475 BUG_ON(iop_chan_is_busy(iop_chan)); iop_chan_start_null_memcpy() 1478 iop_adma_device_clear_err_status(iop_chan); iop_chan_start_null_memcpy() 1481 iop_chan_disable(iop_chan); iop_chan_start_null_memcpy() 1484 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); iop_chan_start_null_memcpy() 1492 iop_chan_enable(iop_chan); iop_chan_start_null_memcpy() 1494 dev_err(iop_chan->device->common.dev, iop_chan_start_null_memcpy() 1496 spin_unlock_bh(&iop_chan->lock); iop_chan_start_null_memcpy() 1499 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) iop_chan_start_null_xor() argument 1505 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); iop_chan_start_null_xor() 1507 spin_lock_bh(&iop_chan->lock); iop_chan_start_null_xor() 1509 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); iop_chan_start_null_xor() 1512 list_splice_init(&sw_desc->tx_list, &iop_chan->chain); iop_chan_start_null_xor() 1515 iop_desc_set_byte_count(grp_start, iop_chan, 0); iop_chan_start_null_xor() 1516 iop_desc_set_dest_addr(grp_start, iop_chan, 0); iop_chan_start_null_xor() 1525 iop_chan->common.completed_cookie = cookie - 1; iop_chan_start_null_xor() 1528 BUG_ON(iop_chan_is_busy(iop_chan)); iop_chan_start_null_xor() 1531 iop_adma_device_clear_err_status(iop_chan); iop_chan_start_null_xor() 1534 iop_chan_disable(iop_chan); iop_chan_start_null_xor() 1537 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); iop_chan_start_null_xor() 1545 iop_chan_enable(iop_chan); iop_chan_start_null_xor() 1547 dev_err(iop_chan->device->common.dev, iop_chan_start_null_xor() 1549 spin_unlock_bh(&iop_chan->lock); iop_chan_start_null_xor() 61 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, struct iop_adma_chan *iop_chan, dma_cookie_t cookie) iop_adma_run_tx_complete_actions() argument 89 iop_adma_clean_slot(struct iop_adma_desc_slot *desc, struct iop_adma_chan *iop_chan) iop_adma_clean_slot() argument
|