1/* 2 3 Broadcom B43 wireless driver 4 5 DMA ringbuffer and descriptor allocation/management 6 7 Copyright (c) 2005, 2006 Michael Buesch <m@bues.ch> 8 9 Some code in this file is derived from the b44.c driver 10 Copyright (C) 2002 David S. Miller 11 Copyright (C) Pekka Pietikainen 12 13 This program is free software; you can redistribute it and/or modify 14 it under the terms of the GNU General Public License as published by 15 the Free Software Foundation; either version 2 of the License, or 16 (at your option) any later version. 17 18 This program is distributed in the hope that it will be useful, 19 but WITHOUT ANY WARRANTY; without even the implied warranty of 20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 GNU General Public License for more details. 22 23 You should have received a copy of the GNU General Public License 24 along with this program; see the file COPYING. If not, write to 25 the Free Software Foundation, Inc., 51 Franklin Steet, Fifth Floor, 26 Boston, MA 02110-1301, USA. 27 28*/ 29 30#include "b43.h" 31#include "dma.h" 32#include "main.h" 33#include "debugfs.h" 34#include "xmit.h" 35 36#include <linux/dma-mapping.h> 37#include <linux/pci.h> 38#include <linux/delay.h> 39#include <linux/skbuff.h> 40#include <linux/etherdevice.h> 41#include <linux/slab.h> 42#include <asm/div64.h> 43 44 45/* Required number of TX DMA slots per TX frame. 46 * This currently is 2, because we put the header and the ieee80211 frame 47 * into separate slots. */ 48#define TX_SLOTS_PER_FRAME 2 49 50static u32 b43_dma_address(struct b43_dma *dma, dma_addr_t dmaaddr, 51 enum b43_addrtype addrtype) 52{ 53 u32 uninitialized_var(addr); 54 55 switch (addrtype) { 56 case B43_DMA_ADDR_LOW: 57 addr = lower_32_bits(dmaaddr); 58 if (dma->translation_in_low) { 59 addr &= ~SSB_DMA_TRANSLATION_MASK; 60 addr |= dma->translation; 61 } 62 break; 63 case B43_DMA_ADDR_HIGH: 64 addr = upper_32_bits(dmaaddr); 65 if (!dma->translation_in_low) { 66 addr &= ~SSB_DMA_TRANSLATION_MASK; 67 addr |= dma->translation; 68 } 69 break; 70 case B43_DMA_ADDR_EXT: 71 if (dma->translation_in_low) 72 addr = lower_32_bits(dmaaddr); 73 else 74 addr = upper_32_bits(dmaaddr); 75 addr &= SSB_DMA_TRANSLATION_MASK; 76 addr >>= SSB_DMA_TRANSLATION_SHIFT; 77 break; 78 } 79 80 return addr; 81} 82 83/* 32bit DMA ops. */ 84static 85struct b43_dmadesc_generic *op32_idx2desc(struct b43_dmaring *ring, 86 int slot, 87 struct b43_dmadesc_meta **meta) 88{ 89 struct b43_dmadesc32 *desc; 90 91 *meta = &(ring->meta[slot]); 92 desc = ring->descbase; 93 desc = &(desc[slot]); 94 95 return (struct b43_dmadesc_generic *)desc; 96} 97 98static void op32_fill_descriptor(struct b43_dmaring *ring, 99 struct b43_dmadesc_generic *desc, 100 dma_addr_t dmaaddr, u16 bufsize, 101 int start, int end, int irq) 102{ 103 struct b43_dmadesc32 *descbase = ring->descbase; 104 int slot; 105 u32 ctl; 106 u32 addr; 107 u32 addrext; 108 109 slot = (int)(&(desc->dma32) - descbase); 110 B43_WARN_ON(!(slot >= 0 && slot < ring->nr_slots)); 111 112 addr = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_LOW); 113 addrext = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_EXT); 114 115 ctl = bufsize & B43_DMA32_DCTL_BYTECNT; 116 if (slot == ring->nr_slots - 1) 117 ctl |= B43_DMA32_DCTL_DTABLEEND; 118 if (start) 119 ctl |= B43_DMA32_DCTL_FRAMESTART; 120 if (end) 121 ctl |= B43_DMA32_DCTL_FRAMEEND; 122 if (irq) 123 ctl |= B43_DMA32_DCTL_IRQ; 124 ctl |= (addrext << B43_DMA32_DCTL_ADDREXT_SHIFT) 125 & B43_DMA32_DCTL_ADDREXT_MASK; 126 127 desc->dma32.control = cpu_to_le32(ctl); 128 desc->dma32.address = cpu_to_le32(addr); 129} 130 131static void op32_poke_tx(struct b43_dmaring *ring, int slot) 132{ 133 b43_dma_write(ring, B43_DMA32_TXINDEX, 134 (u32) (slot * sizeof(struct b43_dmadesc32))); 135} 136 137static void op32_tx_suspend(struct b43_dmaring *ring) 138{ 139 b43_dma_write(ring, B43_DMA32_TXCTL, b43_dma_read(ring, B43_DMA32_TXCTL) 140 | B43_DMA32_TXSUSPEND); 141} 142 143static void op32_tx_resume(struct b43_dmaring *ring) 144{ 145 b43_dma_write(ring, B43_DMA32_TXCTL, b43_dma_read(ring, B43_DMA32_TXCTL) 146 & ~B43_DMA32_TXSUSPEND); 147} 148 149static int op32_get_current_rxslot(struct b43_dmaring *ring) 150{ 151 u32 val; 152 153 val = b43_dma_read(ring, B43_DMA32_RXSTATUS); 154 val &= B43_DMA32_RXDPTR; 155 156 return (val / sizeof(struct b43_dmadesc32)); 157} 158 159static void op32_set_current_rxslot(struct b43_dmaring *ring, int slot) 160{ 161 b43_dma_write(ring, B43_DMA32_RXINDEX, 162 (u32) (slot * sizeof(struct b43_dmadesc32))); 163} 164 165static const struct b43_dma_ops dma32_ops = { 166 .idx2desc = op32_idx2desc, 167 .fill_descriptor = op32_fill_descriptor, 168 .poke_tx = op32_poke_tx, 169 .tx_suspend = op32_tx_suspend, 170 .tx_resume = op32_tx_resume, 171 .get_current_rxslot = op32_get_current_rxslot, 172 .set_current_rxslot = op32_set_current_rxslot, 173}; 174 175/* 64bit DMA ops. */ 176static 177struct b43_dmadesc_generic *op64_idx2desc(struct b43_dmaring *ring, 178 int slot, 179 struct b43_dmadesc_meta **meta) 180{ 181 struct b43_dmadesc64 *desc; 182 183 *meta = &(ring->meta[slot]); 184 desc = ring->descbase; 185 desc = &(desc[slot]); 186 187 return (struct b43_dmadesc_generic *)desc; 188} 189 190static void op64_fill_descriptor(struct b43_dmaring *ring, 191 struct b43_dmadesc_generic *desc, 192 dma_addr_t dmaaddr, u16 bufsize, 193 int start, int end, int irq) 194{ 195 struct b43_dmadesc64 *descbase = ring->descbase; 196 int slot; 197 u32 ctl0 = 0, ctl1 = 0; 198 u32 addrlo, addrhi; 199 u32 addrext; 200 201 slot = (int)(&(desc->dma64) - descbase); 202 B43_WARN_ON(!(slot >= 0 && slot < ring->nr_slots)); 203 204 addrlo = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_LOW); 205 addrhi = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_HIGH); 206 addrext = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_EXT); 207 208 if (slot == ring->nr_slots - 1) 209 ctl0 |= B43_DMA64_DCTL0_DTABLEEND; 210 if (start) 211 ctl0 |= B43_DMA64_DCTL0_FRAMESTART; 212 if (end) 213 ctl0 |= B43_DMA64_DCTL0_FRAMEEND; 214 if (irq) 215 ctl0 |= B43_DMA64_DCTL0_IRQ; 216 ctl1 |= bufsize & B43_DMA64_DCTL1_BYTECNT; 217 ctl1 |= (addrext << B43_DMA64_DCTL1_ADDREXT_SHIFT) 218 & B43_DMA64_DCTL1_ADDREXT_MASK; 219 220 desc->dma64.control0 = cpu_to_le32(ctl0); 221 desc->dma64.control1 = cpu_to_le32(ctl1); 222 desc->dma64.address_low = cpu_to_le32(addrlo); 223 desc->dma64.address_high = cpu_to_le32(addrhi); 224} 225 226static void op64_poke_tx(struct b43_dmaring *ring, int slot) 227{ 228 b43_dma_write(ring, B43_DMA64_TXINDEX, 229 (u32) (slot * sizeof(struct b43_dmadesc64))); 230} 231 232static void op64_tx_suspend(struct b43_dmaring *ring) 233{ 234 b43_dma_write(ring, B43_DMA64_TXCTL, b43_dma_read(ring, B43_DMA64_TXCTL) 235 | B43_DMA64_TXSUSPEND); 236} 237 238static void op64_tx_resume(struct b43_dmaring *ring) 239{ 240 b43_dma_write(ring, B43_DMA64_TXCTL, b43_dma_read(ring, B43_DMA64_TXCTL) 241 & ~B43_DMA64_TXSUSPEND); 242} 243 244static int op64_get_current_rxslot(struct b43_dmaring *ring) 245{ 246 u32 val; 247 248 val = b43_dma_read(ring, B43_DMA64_RXSTATUS); 249 val &= B43_DMA64_RXSTATDPTR; 250 251 return (val / sizeof(struct b43_dmadesc64)); 252} 253 254static void op64_set_current_rxslot(struct b43_dmaring *ring, int slot) 255{ 256 b43_dma_write(ring, B43_DMA64_RXINDEX, 257 (u32) (slot * sizeof(struct b43_dmadesc64))); 258} 259 260static const struct b43_dma_ops dma64_ops = { 261 .idx2desc = op64_idx2desc, 262 .fill_descriptor = op64_fill_descriptor, 263 .poke_tx = op64_poke_tx, 264 .tx_suspend = op64_tx_suspend, 265 .tx_resume = op64_tx_resume, 266 .get_current_rxslot = op64_get_current_rxslot, 267 .set_current_rxslot = op64_set_current_rxslot, 268}; 269 270static inline int free_slots(struct b43_dmaring *ring) 271{ 272 return (ring->nr_slots - ring->used_slots); 273} 274 275static inline int next_slot(struct b43_dmaring *ring, int slot) 276{ 277 B43_WARN_ON(!(slot >= -1 && slot <= ring->nr_slots - 1)); 278 if (slot == ring->nr_slots - 1) 279 return 0; 280 return slot + 1; 281} 282 283static inline int prev_slot(struct b43_dmaring *ring, int slot) 284{ 285 B43_WARN_ON(!(slot >= 0 && slot <= ring->nr_slots - 1)); 286 if (slot == 0) 287 return ring->nr_slots - 1; 288 return slot - 1; 289} 290 291#ifdef CONFIG_B43_DEBUG 292static void update_max_used_slots(struct b43_dmaring *ring, 293 int current_used_slots) 294{ 295 if (current_used_slots <= ring->max_used_slots) 296 return; 297 ring->max_used_slots = current_used_slots; 298 if (b43_debug(ring->dev, B43_DBG_DMAVERBOSE)) { 299 b43dbg(ring->dev->wl, 300 "max_used_slots increased to %d on %s ring %d\n", 301 ring->max_used_slots, 302 ring->tx ? "TX" : "RX", ring->index); 303 } 304} 305#else 306static inline 307 void update_max_used_slots(struct b43_dmaring *ring, int current_used_slots) 308{ 309} 310#endif /* DEBUG */ 311 312/* Request a slot for usage. */ 313static inline int request_slot(struct b43_dmaring *ring) 314{ 315 int slot; 316 317 B43_WARN_ON(!ring->tx); 318 B43_WARN_ON(ring->stopped); 319 B43_WARN_ON(free_slots(ring) == 0); 320 321 slot = next_slot(ring, ring->current_slot); 322 ring->current_slot = slot; 323 ring->used_slots++; 324 325 update_max_used_slots(ring, ring->used_slots); 326 327 return slot; 328} 329 330static u16 b43_dmacontroller_base(enum b43_dmatype type, int controller_idx) 331{ 332 static const u16 map64[] = { 333 B43_MMIO_DMA64_BASE0, 334 B43_MMIO_DMA64_BASE1, 335 B43_MMIO_DMA64_BASE2, 336 B43_MMIO_DMA64_BASE3, 337 B43_MMIO_DMA64_BASE4, 338 B43_MMIO_DMA64_BASE5, 339 }; 340 static const u16 map32[] = { 341 B43_MMIO_DMA32_BASE0, 342 B43_MMIO_DMA32_BASE1, 343 B43_MMIO_DMA32_BASE2, 344 B43_MMIO_DMA32_BASE3, 345 B43_MMIO_DMA32_BASE4, 346 B43_MMIO_DMA32_BASE5, 347 }; 348 349 if (type == B43_DMA_64BIT) { 350 B43_WARN_ON(!(controller_idx >= 0 && 351 controller_idx < ARRAY_SIZE(map64))); 352 return map64[controller_idx]; 353 } 354 B43_WARN_ON(!(controller_idx >= 0 && 355 controller_idx < ARRAY_SIZE(map32))); 356 return map32[controller_idx]; 357} 358 359static inline 360 dma_addr_t map_descbuffer(struct b43_dmaring *ring, 361 unsigned char *buf, size_t len, int tx) 362{ 363 dma_addr_t dmaaddr; 364 365 if (tx) { 366 dmaaddr = dma_map_single(ring->dev->dev->dma_dev, 367 buf, len, DMA_TO_DEVICE); 368 } else { 369 dmaaddr = dma_map_single(ring->dev->dev->dma_dev, 370 buf, len, DMA_FROM_DEVICE); 371 } 372 373 return dmaaddr; 374} 375 376static inline 377 void unmap_descbuffer(struct b43_dmaring *ring, 378 dma_addr_t addr, size_t len, int tx) 379{ 380 if (tx) { 381 dma_unmap_single(ring->dev->dev->dma_dev, 382 addr, len, DMA_TO_DEVICE); 383 } else { 384 dma_unmap_single(ring->dev->dev->dma_dev, 385 addr, len, DMA_FROM_DEVICE); 386 } 387} 388 389static inline 390 void sync_descbuffer_for_cpu(struct b43_dmaring *ring, 391 dma_addr_t addr, size_t len) 392{ 393 B43_WARN_ON(ring->tx); 394 dma_sync_single_for_cpu(ring->dev->dev->dma_dev, 395 addr, len, DMA_FROM_DEVICE); 396} 397 398static inline 399 void sync_descbuffer_for_device(struct b43_dmaring *ring, 400 dma_addr_t addr, size_t len) 401{ 402 B43_WARN_ON(ring->tx); 403 dma_sync_single_for_device(ring->dev->dev->dma_dev, 404 addr, len, DMA_FROM_DEVICE); 405} 406 407static inline 408 void free_descriptor_buffer(struct b43_dmaring *ring, 409 struct b43_dmadesc_meta *meta) 410{ 411 if (meta->skb) { 412 if (ring->tx) 413 ieee80211_free_txskb(ring->dev->wl->hw, meta->skb); 414 else 415 dev_kfree_skb_any(meta->skb); 416 meta->skb = NULL; 417 } 418} 419 420static int alloc_ringmemory(struct b43_dmaring *ring) 421{ 422 /* The specs call for 4K buffers for 30- and 32-bit DMA with 4K 423 * alignment and 8K buffers for 64-bit DMA with 8K alignment. 424 * In practice we could use smaller buffers for the latter, but the 425 * alignment is really important because of the hardware bug. If bit 426 * 0x00001000 is used in DMA address, some hardware (like BCM4331) 427 * copies that bit into B43_DMA64_RXSTATUS and we get false values from 428 * B43_DMA64_RXSTATDPTR. Let's just use 8K buffers even if we don't use 429 * more than 256 slots for ring. 430 */ 431 u16 ring_mem_size = (ring->type == B43_DMA_64BIT) ? 432 B43_DMA64_RINGMEMSIZE : B43_DMA32_RINGMEMSIZE; 433 434 ring->descbase = dma_zalloc_coherent(ring->dev->dev->dma_dev, 435 ring_mem_size, &(ring->dmabase), 436 GFP_KERNEL); 437 if (!ring->descbase) 438 return -ENOMEM; 439 440 return 0; 441} 442 443static void free_ringmemory(struct b43_dmaring *ring) 444{ 445 u16 ring_mem_size = (ring->type == B43_DMA_64BIT) ? 446 B43_DMA64_RINGMEMSIZE : B43_DMA32_RINGMEMSIZE; 447 dma_free_coherent(ring->dev->dev->dma_dev, ring_mem_size, 448 ring->descbase, ring->dmabase); 449} 450 451/* Reset the RX DMA channel */ 452static int b43_dmacontroller_rx_reset(struct b43_wldev *dev, u16 mmio_base, 453 enum b43_dmatype type) 454{ 455 int i; 456 u32 value; 457 u16 offset; 458 459 might_sleep(); 460 461 offset = (type == B43_DMA_64BIT) ? B43_DMA64_RXCTL : B43_DMA32_RXCTL; 462 b43_write32(dev, mmio_base + offset, 0); 463 for (i = 0; i < 10; i++) { 464 offset = (type == B43_DMA_64BIT) ? B43_DMA64_RXSTATUS : 465 B43_DMA32_RXSTATUS; 466 value = b43_read32(dev, mmio_base + offset); 467 if (type == B43_DMA_64BIT) { 468 value &= B43_DMA64_RXSTAT; 469 if (value == B43_DMA64_RXSTAT_DISABLED) { 470 i = -1; 471 break; 472 } 473 } else { 474 value &= B43_DMA32_RXSTATE; 475 if (value == B43_DMA32_RXSTAT_DISABLED) { 476 i = -1; 477 break; 478 } 479 } 480 msleep(1); 481 } 482 if (i != -1) { 483 b43err(dev->wl, "DMA RX reset timed out\n"); 484 return -ENODEV; 485 } 486 487 return 0; 488} 489 490/* Reset the TX DMA channel */ 491static int b43_dmacontroller_tx_reset(struct b43_wldev *dev, u16 mmio_base, 492 enum b43_dmatype type) 493{ 494 int i; 495 u32 value; 496 u16 offset; 497 498 might_sleep(); 499 500 for (i = 0; i < 10; i++) { 501 offset = (type == B43_DMA_64BIT) ? B43_DMA64_TXSTATUS : 502 B43_DMA32_TXSTATUS; 503 value = b43_read32(dev, mmio_base + offset); 504 if (type == B43_DMA_64BIT) { 505 value &= B43_DMA64_TXSTAT; 506 if (value == B43_DMA64_TXSTAT_DISABLED || 507 value == B43_DMA64_TXSTAT_IDLEWAIT || 508 value == B43_DMA64_TXSTAT_STOPPED) 509 break; 510 } else { 511 value &= B43_DMA32_TXSTATE; 512 if (value == B43_DMA32_TXSTAT_DISABLED || 513 value == B43_DMA32_TXSTAT_IDLEWAIT || 514 value == B43_DMA32_TXSTAT_STOPPED) 515 break; 516 } 517 msleep(1); 518 } 519 offset = (type == B43_DMA_64BIT) ? B43_DMA64_TXCTL : B43_DMA32_TXCTL; 520 b43_write32(dev, mmio_base + offset, 0); 521 for (i = 0; i < 10; i++) { 522 offset = (type == B43_DMA_64BIT) ? B43_DMA64_TXSTATUS : 523 B43_DMA32_TXSTATUS; 524 value = b43_read32(dev, mmio_base + offset); 525 if (type == B43_DMA_64BIT) { 526 value &= B43_DMA64_TXSTAT; 527 if (value == B43_DMA64_TXSTAT_DISABLED) { 528 i = -1; 529 break; 530 } 531 } else { 532 value &= B43_DMA32_TXSTATE; 533 if (value == B43_DMA32_TXSTAT_DISABLED) { 534 i = -1; 535 break; 536 } 537 } 538 msleep(1); 539 } 540 if (i != -1) { 541 b43err(dev->wl, "DMA TX reset timed out\n"); 542 return -ENODEV; 543 } 544 /* ensure the reset is completed. */ 545 msleep(1); 546 547 return 0; 548} 549 550/* Check if a DMA mapping address is invalid. */ 551static bool b43_dma_mapping_error(struct b43_dmaring *ring, 552 dma_addr_t addr, 553 size_t buffersize, bool dma_to_device) 554{ 555 if (unlikely(dma_mapping_error(ring->dev->dev->dma_dev, addr))) 556 return true; 557 558 switch (ring->type) { 559 case B43_DMA_30BIT: 560 if ((u64)addr + buffersize > (1ULL << 30)) 561 goto address_error; 562 break; 563 case B43_DMA_32BIT: 564 if ((u64)addr + buffersize > (1ULL << 32)) 565 goto address_error; 566 break; 567 case B43_DMA_64BIT: 568 /* Currently we can't have addresses beyond 569 * 64bit in the kernel. */ 570 break; 571 } 572 573 /* The address is OK. */ 574 return false; 575 576address_error: 577 /* We can't support this address. Unmap it again. */ 578 unmap_descbuffer(ring, addr, buffersize, dma_to_device); 579 580 return true; 581} 582 583static bool b43_rx_buffer_is_poisoned(struct b43_dmaring *ring, struct sk_buff *skb) 584{ 585 unsigned char *f = skb->data + ring->frameoffset; 586 587 return ((f[0] & f[1] & f[2] & f[3] & f[4] & f[5] & f[6] & f[7]) == 0xFF); 588} 589 590static void b43_poison_rx_buffer(struct b43_dmaring *ring, struct sk_buff *skb) 591{ 592 struct b43_rxhdr_fw4 *rxhdr; 593 unsigned char *frame; 594 595 /* This poisons the RX buffer to detect DMA failures. */ 596 597 rxhdr = (struct b43_rxhdr_fw4 *)(skb->data); 598 rxhdr->frame_len = 0; 599 600 B43_WARN_ON(ring->rx_buffersize < ring->frameoffset + sizeof(struct b43_plcp_hdr6) + 2); 601 frame = skb->data + ring->frameoffset; 602 memset(frame, 0xFF, sizeof(struct b43_plcp_hdr6) + 2 /* padding */); 603} 604 605static int setup_rx_descbuffer(struct b43_dmaring *ring, 606 struct b43_dmadesc_generic *desc, 607 struct b43_dmadesc_meta *meta, gfp_t gfp_flags) 608{ 609 dma_addr_t dmaaddr; 610 struct sk_buff *skb; 611 612 B43_WARN_ON(ring->tx); 613 614 skb = __dev_alloc_skb(ring->rx_buffersize, gfp_flags); 615 if (unlikely(!skb)) 616 return -ENOMEM; 617 b43_poison_rx_buffer(ring, skb); 618 dmaaddr = map_descbuffer(ring, skb->data, ring->rx_buffersize, 0); 619 if (b43_dma_mapping_error(ring, dmaaddr, ring->rx_buffersize, 0)) { 620 /* ugh. try to realloc in zone_dma */ 621 gfp_flags |= GFP_DMA; 622 623 dev_kfree_skb_any(skb); 624 625 skb = __dev_alloc_skb(ring->rx_buffersize, gfp_flags); 626 if (unlikely(!skb)) 627 return -ENOMEM; 628 b43_poison_rx_buffer(ring, skb); 629 dmaaddr = map_descbuffer(ring, skb->data, 630 ring->rx_buffersize, 0); 631 if (b43_dma_mapping_error(ring, dmaaddr, ring->rx_buffersize, 0)) { 632 b43err(ring->dev->wl, "RX DMA buffer allocation failed\n"); 633 dev_kfree_skb_any(skb); 634 return -EIO; 635 } 636 } 637 638 meta->skb = skb; 639 meta->dmaaddr = dmaaddr; 640 ring->ops->fill_descriptor(ring, desc, dmaaddr, 641 ring->rx_buffersize, 0, 0, 0); 642 643 return 0; 644} 645 646/* Allocate the initial descbuffers. 647 * This is used for an RX ring only. 648 */ 649static int alloc_initial_descbuffers(struct b43_dmaring *ring) 650{ 651 int i, err = -ENOMEM; 652 struct b43_dmadesc_generic *desc; 653 struct b43_dmadesc_meta *meta; 654 655 for (i = 0; i < ring->nr_slots; i++) { 656 desc = ring->ops->idx2desc(ring, i, &meta); 657 658 err = setup_rx_descbuffer(ring, desc, meta, GFP_KERNEL); 659 if (err) { 660 b43err(ring->dev->wl, 661 "Failed to allocate initial descbuffers\n"); 662 goto err_unwind; 663 } 664 } 665 mb(); 666 ring->used_slots = ring->nr_slots; 667 err = 0; 668 out: 669 return err; 670 671 err_unwind: 672 for (i--; i >= 0; i--) { 673 desc = ring->ops->idx2desc(ring, i, &meta); 674 675 unmap_descbuffer(ring, meta->dmaaddr, ring->rx_buffersize, 0); 676 dev_kfree_skb(meta->skb); 677 } 678 goto out; 679} 680 681/* Do initial setup of the DMA controller. 682 * Reset the controller, write the ring busaddress 683 * and switch the "enable" bit on. 684 */ 685static int dmacontroller_setup(struct b43_dmaring *ring) 686{ 687 int err = 0; 688 u32 value; 689 u32 addrext; 690 bool parity = ring->dev->dma.parity; 691 u32 addrlo; 692 u32 addrhi; 693 694 if (ring->tx) { 695 if (ring->type == B43_DMA_64BIT) { 696 u64 ringbase = (u64) (ring->dmabase); 697 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 698 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 699 addrhi = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_HIGH); 700 701 value = B43_DMA64_TXENABLE; 702 value |= (addrext << B43_DMA64_TXADDREXT_SHIFT) 703 & B43_DMA64_TXADDREXT_MASK; 704 if (!parity) 705 value |= B43_DMA64_TXPARITYDISABLE; 706 b43_dma_write(ring, B43_DMA64_TXCTL, value); 707 b43_dma_write(ring, B43_DMA64_TXRINGLO, addrlo); 708 b43_dma_write(ring, B43_DMA64_TXRINGHI, addrhi); 709 } else { 710 u32 ringbase = (u32) (ring->dmabase); 711 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 712 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 713 714 value = B43_DMA32_TXENABLE; 715 value |= (addrext << B43_DMA32_TXADDREXT_SHIFT) 716 & B43_DMA32_TXADDREXT_MASK; 717 if (!parity) 718 value |= B43_DMA32_TXPARITYDISABLE; 719 b43_dma_write(ring, B43_DMA32_TXCTL, value); 720 b43_dma_write(ring, B43_DMA32_TXRING, addrlo); 721 } 722 } else { 723 err = alloc_initial_descbuffers(ring); 724 if (err) 725 goto out; 726 if (ring->type == B43_DMA_64BIT) { 727 u64 ringbase = (u64) (ring->dmabase); 728 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 729 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 730 addrhi = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_HIGH); 731 732 value = (ring->frameoffset << B43_DMA64_RXFROFF_SHIFT); 733 value |= B43_DMA64_RXENABLE; 734 value |= (addrext << B43_DMA64_RXADDREXT_SHIFT) 735 & B43_DMA64_RXADDREXT_MASK; 736 if (!parity) 737 value |= B43_DMA64_RXPARITYDISABLE; 738 b43_dma_write(ring, B43_DMA64_RXCTL, value); 739 b43_dma_write(ring, B43_DMA64_RXRINGLO, addrlo); 740 b43_dma_write(ring, B43_DMA64_RXRINGHI, addrhi); 741 b43_dma_write(ring, B43_DMA64_RXINDEX, ring->nr_slots * 742 sizeof(struct b43_dmadesc64)); 743 } else { 744 u32 ringbase = (u32) (ring->dmabase); 745 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 746 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 747 748 value = (ring->frameoffset << B43_DMA32_RXFROFF_SHIFT); 749 value |= B43_DMA32_RXENABLE; 750 value |= (addrext << B43_DMA32_RXADDREXT_SHIFT) 751 & B43_DMA32_RXADDREXT_MASK; 752 if (!parity) 753 value |= B43_DMA32_RXPARITYDISABLE; 754 b43_dma_write(ring, B43_DMA32_RXCTL, value); 755 b43_dma_write(ring, B43_DMA32_RXRING, addrlo); 756 b43_dma_write(ring, B43_DMA32_RXINDEX, ring->nr_slots * 757 sizeof(struct b43_dmadesc32)); 758 } 759 } 760 761out: 762 return err; 763} 764 765/* Shutdown the DMA controller. */ 766static void dmacontroller_cleanup(struct b43_dmaring *ring) 767{ 768 if (ring->tx) { 769 b43_dmacontroller_tx_reset(ring->dev, ring->mmio_base, 770 ring->type); 771 if (ring->type == B43_DMA_64BIT) { 772 b43_dma_write(ring, B43_DMA64_TXRINGLO, 0); 773 b43_dma_write(ring, B43_DMA64_TXRINGHI, 0); 774 } else 775 b43_dma_write(ring, B43_DMA32_TXRING, 0); 776 } else { 777 b43_dmacontroller_rx_reset(ring->dev, ring->mmio_base, 778 ring->type); 779 if (ring->type == B43_DMA_64BIT) { 780 b43_dma_write(ring, B43_DMA64_RXRINGLO, 0); 781 b43_dma_write(ring, B43_DMA64_RXRINGHI, 0); 782 } else 783 b43_dma_write(ring, B43_DMA32_RXRING, 0); 784 } 785} 786 787static void free_all_descbuffers(struct b43_dmaring *ring) 788{ 789 struct b43_dmadesc_meta *meta; 790 int i; 791 792 if (!ring->used_slots) 793 return; 794 for (i = 0; i < ring->nr_slots; i++) { 795 /* get meta - ignore returned value */ 796 ring->ops->idx2desc(ring, i, &meta); 797 798 if (!meta->skb || b43_dma_ptr_is_poisoned(meta->skb)) { 799 B43_WARN_ON(!ring->tx); 800 continue; 801 } 802 if (ring->tx) { 803 unmap_descbuffer(ring, meta->dmaaddr, 804 meta->skb->len, 1); 805 } else { 806 unmap_descbuffer(ring, meta->dmaaddr, 807 ring->rx_buffersize, 0); 808 } 809 free_descriptor_buffer(ring, meta); 810 } 811} 812 813static u64 supported_dma_mask(struct b43_wldev *dev) 814{ 815 u32 tmp; 816 u16 mmio_base; 817 818 switch (dev->dev->bus_type) { 819#ifdef CONFIG_B43_BCMA 820 case B43_BUS_BCMA: 821 tmp = bcma_aread32(dev->dev->bdev, BCMA_IOST); 822 if (tmp & BCMA_IOST_DMA64) 823 return DMA_BIT_MASK(64); 824 break; 825#endif 826#ifdef CONFIG_B43_SSB 827 case B43_BUS_SSB: 828 tmp = ssb_read32(dev->dev->sdev, SSB_TMSHIGH); 829 if (tmp & SSB_TMSHIGH_DMA64) 830 return DMA_BIT_MASK(64); 831 break; 832#endif 833 } 834 835 mmio_base = b43_dmacontroller_base(0, 0); 836 b43_write32(dev, mmio_base + B43_DMA32_TXCTL, B43_DMA32_TXADDREXT_MASK); 837 tmp = b43_read32(dev, mmio_base + B43_DMA32_TXCTL); 838 if (tmp & B43_DMA32_TXADDREXT_MASK) 839 return DMA_BIT_MASK(32); 840 841 return DMA_BIT_MASK(30); 842} 843 844static enum b43_dmatype dma_mask_to_engine_type(u64 dmamask) 845{ 846 if (dmamask == DMA_BIT_MASK(30)) 847 return B43_DMA_30BIT; 848 if (dmamask == DMA_BIT_MASK(32)) 849 return B43_DMA_32BIT; 850 if (dmamask == DMA_BIT_MASK(64)) 851 return B43_DMA_64BIT; 852 B43_WARN_ON(1); 853 return B43_DMA_30BIT; 854} 855 856/* Main initialization function. */ 857static 858struct b43_dmaring *b43_setup_dmaring(struct b43_wldev *dev, 859 int controller_index, 860 int for_tx, 861 enum b43_dmatype type) 862{ 863 struct b43_dmaring *ring; 864 int i, err; 865 dma_addr_t dma_test; 866 867 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 868 if (!ring) 869 goto out; 870 871 ring->nr_slots = B43_RXRING_SLOTS; 872 if (for_tx) 873 ring->nr_slots = B43_TXRING_SLOTS; 874 875 ring->meta = kcalloc(ring->nr_slots, sizeof(struct b43_dmadesc_meta), 876 GFP_KERNEL); 877 if (!ring->meta) 878 goto err_kfree_ring; 879 for (i = 0; i < ring->nr_slots; i++) 880 ring->meta->skb = B43_DMA_PTR_POISON; 881 882 ring->type = type; 883 ring->dev = dev; 884 ring->mmio_base = b43_dmacontroller_base(type, controller_index); 885 ring->index = controller_index; 886 if (type == B43_DMA_64BIT) 887 ring->ops = &dma64_ops; 888 else 889 ring->ops = &dma32_ops; 890 if (for_tx) { 891 ring->tx = true; 892 ring->current_slot = -1; 893 } else { 894 if (ring->index == 0) { 895 switch (dev->fw.hdr_format) { 896 case B43_FW_HDR_598: 897 ring->rx_buffersize = B43_DMA0_RX_FW598_BUFSIZE; 898 ring->frameoffset = B43_DMA0_RX_FW598_FO; 899 break; 900 case B43_FW_HDR_410: 901 case B43_FW_HDR_351: 902 ring->rx_buffersize = B43_DMA0_RX_FW351_BUFSIZE; 903 ring->frameoffset = B43_DMA0_RX_FW351_FO; 904 break; 905 } 906 } else 907 B43_WARN_ON(1); 908 } 909#ifdef CONFIG_B43_DEBUG 910 ring->last_injected_overflow = jiffies; 911#endif 912 913 if (for_tx) { 914 /* Assumption: B43_TXRING_SLOTS can be divided by TX_SLOTS_PER_FRAME */ 915 BUILD_BUG_ON(B43_TXRING_SLOTS % TX_SLOTS_PER_FRAME != 0); 916 917 ring->txhdr_cache = kcalloc(ring->nr_slots / TX_SLOTS_PER_FRAME, 918 b43_txhdr_size(dev), 919 GFP_KERNEL); 920 if (!ring->txhdr_cache) 921 goto err_kfree_meta; 922 923 /* test for ability to dma to txhdr_cache */ 924 dma_test = dma_map_single(dev->dev->dma_dev, 925 ring->txhdr_cache, 926 b43_txhdr_size(dev), 927 DMA_TO_DEVICE); 928 929 if (b43_dma_mapping_error(ring, dma_test, 930 b43_txhdr_size(dev), 1)) { 931 /* ugh realloc */ 932 kfree(ring->txhdr_cache); 933 ring->txhdr_cache = kcalloc(ring->nr_slots / TX_SLOTS_PER_FRAME, 934 b43_txhdr_size(dev), 935 GFP_KERNEL | GFP_DMA); 936 if (!ring->txhdr_cache) 937 goto err_kfree_meta; 938 939 dma_test = dma_map_single(dev->dev->dma_dev, 940 ring->txhdr_cache, 941 b43_txhdr_size(dev), 942 DMA_TO_DEVICE); 943 944 if (b43_dma_mapping_error(ring, dma_test, 945 b43_txhdr_size(dev), 1)) { 946 947 b43err(dev->wl, 948 "TXHDR DMA allocation failed\n"); 949 goto err_kfree_txhdr_cache; 950 } 951 } 952 953 dma_unmap_single(dev->dev->dma_dev, 954 dma_test, b43_txhdr_size(dev), 955 DMA_TO_DEVICE); 956 } 957 958 err = alloc_ringmemory(ring); 959 if (err) 960 goto err_kfree_txhdr_cache; 961 err = dmacontroller_setup(ring); 962 if (err) 963 goto err_free_ringmemory; 964 965 out: 966 return ring; 967 968 err_free_ringmemory: 969 free_ringmemory(ring); 970 err_kfree_txhdr_cache: 971 kfree(ring->txhdr_cache); 972 err_kfree_meta: 973 kfree(ring->meta); 974 err_kfree_ring: 975 kfree(ring); 976 ring = NULL; 977 goto out; 978} 979 980#define divide(a, b) ({ \ 981 typeof(a) __a = a; \ 982 do_div(__a, b); \ 983 __a; \ 984 }) 985 986#define modulo(a, b) ({ \ 987 typeof(a) __a = a; \ 988 do_div(__a, b); \ 989 }) 990 991/* Main cleanup function. */ 992static void b43_destroy_dmaring(struct b43_dmaring *ring, 993 const char *ringname) 994{ 995 if (!ring) 996 return; 997 998#ifdef CONFIG_B43_DEBUG 999 { 1000 /* Print some statistics. */ 1001 u64 failed_packets = ring->nr_failed_tx_packets; 1002 u64 succeed_packets = ring->nr_succeed_tx_packets; 1003 u64 nr_packets = failed_packets + succeed_packets; 1004 u64 permille_failed = 0, average_tries = 0; 1005 1006 if (nr_packets) 1007 permille_failed = divide(failed_packets * 1000, nr_packets); 1008 if (nr_packets) 1009 average_tries = divide(ring->nr_total_packet_tries * 100, nr_packets); 1010 1011 b43dbg(ring->dev->wl, "DMA-%u %s: " 1012 "Used slots %d/%d, Failed frames %llu/%llu = %llu.%01llu%%, " 1013 "Average tries %llu.%02llu\n", 1014 (unsigned int)(ring->type), ringname, 1015 ring->max_used_slots, 1016 ring->nr_slots, 1017 (unsigned long long)failed_packets, 1018 (unsigned long long)nr_packets, 1019 (unsigned long long)divide(permille_failed, 10), 1020 (unsigned long long)modulo(permille_failed, 10), 1021 (unsigned long long)divide(average_tries, 100), 1022 (unsigned long long)modulo(average_tries, 100)); 1023 } 1024#endif /* DEBUG */ 1025 1026 /* Device IRQs are disabled prior entering this function, 1027 * so no need to take care of concurrency with rx handler stuff. 1028 */ 1029 dmacontroller_cleanup(ring); 1030 free_all_descbuffers(ring); 1031 free_ringmemory(ring); 1032 1033 kfree(ring->txhdr_cache); 1034 kfree(ring->meta); 1035 kfree(ring); 1036} 1037 1038#define destroy_ring(dma, ring) do { \ 1039 b43_destroy_dmaring((dma)->ring, __stringify(ring)); \ 1040 (dma)->ring = NULL; \ 1041 } while (0) 1042 1043void b43_dma_free(struct b43_wldev *dev) 1044{ 1045 struct b43_dma *dma; 1046 1047 if (b43_using_pio_transfers(dev)) 1048 return; 1049 dma = &dev->dma; 1050 1051 destroy_ring(dma, rx_ring); 1052 destroy_ring(dma, tx_ring_AC_BK); 1053 destroy_ring(dma, tx_ring_AC_BE); 1054 destroy_ring(dma, tx_ring_AC_VI); 1055 destroy_ring(dma, tx_ring_AC_VO); 1056 destroy_ring(dma, tx_ring_mcast); 1057} 1058 1059static int b43_dma_set_mask(struct b43_wldev *dev, u64 mask) 1060{ 1061 u64 orig_mask = mask; 1062 bool fallback = false; 1063 int err; 1064 1065 /* Try to set the DMA mask. If it fails, try falling back to a 1066 * lower mask, as we can always also support a lower one. */ 1067 while (1) { 1068 err = dma_set_mask_and_coherent(dev->dev->dma_dev, mask); 1069 if (!err) 1070 break; 1071 if (mask == DMA_BIT_MASK(64)) { 1072 mask = DMA_BIT_MASK(32); 1073 fallback = true; 1074 continue; 1075 } 1076 if (mask == DMA_BIT_MASK(32)) { 1077 mask = DMA_BIT_MASK(30); 1078 fallback = true; 1079 continue; 1080 } 1081 b43err(dev->wl, "The machine/kernel does not support " 1082 "the required %u-bit DMA mask\n", 1083 (unsigned int)dma_mask_to_engine_type(orig_mask)); 1084 return -EOPNOTSUPP; 1085 } 1086 if (fallback) { 1087 b43info(dev->wl, "DMA mask fallback from %u-bit to %u-bit\n", 1088 (unsigned int)dma_mask_to_engine_type(orig_mask), 1089 (unsigned int)dma_mask_to_engine_type(mask)); 1090 } 1091 1092 return 0; 1093} 1094 1095/* Some hardware with 64-bit DMA seems to be bugged and looks for translation 1096 * bit in low address word instead of high one. 1097 */ 1098static bool b43_dma_translation_in_low_word(struct b43_wldev *dev, 1099 enum b43_dmatype type) 1100{ 1101 if (type != B43_DMA_64BIT) 1102 return true; 1103 1104#ifdef CONFIG_B43_SSB 1105 if (dev->dev->bus_type == B43_BUS_SSB && 1106 dev->dev->sdev->bus->bustype == SSB_BUSTYPE_PCI && 1107 !(pci_is_pcie(dev->dev->sdev->bus->host_pci) && 1108 ssb_read32(dev->dev->sdev, SSB_TMSHIGH) & SSB_TMSHIGH_DMA64)) 1109 return true; 1110#endif 1111 return false; 1112} 1113 1114int b43_dma_init(struct b43_wldev *dev) 1115{ 1116 struct b43_dma *dma = &dev->dma; 1117 int err; 1118 u64 dmamask; 1119 enum b43_dmatype type; 1120 1121 dmamask = supported_dma_mask(dev); 1122 type = dma_mask_to_engine_type(dmamask); 1123 err = b43_dma_set_mask(dev, dmamask); 1124 if (err) 1125 return err; 1126 1127 switch (dev->dev->bus_type) { 1128#ifdef CONFIG_B43_BCMA 1129 case B43_BUS_BCMA: 1130 dma->translation = bcma_core_dma_translation(dev->dev->bdev); 1131 break; 1132#endif 1133#ifdef CONFIG_B43_SSB 1134 case B43_BUS_SSB: 1135 dma->translation = ssb_dma_translation(dev->dev->sdev); 1136 break; 1137#endif 1138 } 1139 dma->translation_in_low = b43_dma_translation_in_low_word(dev, type); 1140 1141 dma->parity = true; 1142#ifdef CONFIG_B43_BCMA 1143 /* TODO: find out which SSB devices need disabling parity */ 1144 if (dev->dev->bus_type == B43_BUS_BCMA) 1145 dma->parity = false; 1146#endif 1147 1148 err = -ENOMEM; 1149 /* setup TX DMA channels. */ 1150 dma->tx_ring_AC_BK = b43_setup_dmaring(dev, 0, 1, type); 1151 if (!dma->tx_ring_AC_BK) 1152 goto out; 1153 1154 dma->tx_ring_AC_BE = b43_setup_dmaring(dev, 1, 1, type); 1155 if (!dma->tx_ring_AC_BE) 1156 goto err_destroy_bk; 1157 1158 dma->tx_ring_AC_VI = b43_setup_dmaring(dev, 2, 1, type); 1159 if (!dma->tx_ring_AC_VI) 1160 goto err_destroy_be; 1161 1162 dma->tx_ring_AC_VO = b43_setup_dmaring(dev, 3, 1, type); 1163 if (!dma->tx_ring_AC_VO) 1164 goto err_destroy_vi; 1165 1166 dma->tx_ring_mcast = b43_setup_dmaring(dev, 4, 1, type); 1167 if (!dma->tx_ring_mcast) 1168 goto err_destroy_vo; 1169 1170 /* setup RX DMA channel. */ 1171 dma->rx_ring = b43_setup_dmaring(dev, 0, 0, type); 1172 if (!dma->rx_ring) 1173 goto err_destroy_mcast; 1174 1175 /* No support for the TX status DMA ring. */ 1176 B43_WARN_ON(dev->dev->core_rev < 5); 1177 1178 b43dbg(dev->wl, "%u-bit DMA initialized\n", 1179 (unsigned int)type); 1180 err = 0; 1181out: 1182 return err; 1183 1184err_destroy_mcast: 1185 destroy_ring(dma, tx_ring_mcast); 1186err_destroy_vo: 1187 destroy_ring(dma, tx_ring_AC_VO); 1188err_destroy_vi: 1189 destroy_ring(dma, tx_ring_AC_VI); 1190err_destroy_be: 1191 destroy_ring(dma, tx_ring_AC_BE); 1192err_destroy_bk: 1193 destroy_ring(dma, tx_ring_AC_BK); 1194 return err; 1195} 1196 1197/* Generate a cookie for the TX header. */ 1198static u16 generate_cookie(struct b43_dmaring *ring, int slot) 1199{ 1200 u16 cookie; 1201 1202 /* Use the upper 4 bits of the cookie as 1203 * DMA controller ID and store the slot number 1204 * in the lower 12 bits. 1205 * Note that the cookie must never be 0, as this 1206 * is a special value used in RX path. 1207 * It can also not be 0xFFFF because that is special 1208 * for multicast frames. 1209 */ 1210 cookie = (((u16)ring->index + 1) << 12); 1211 B43_WARN_ON(slot & ~0x0FFF); 1212 cookie |= (u16)slot; 1213 1214 return cookie; 1215} 1216 1217/* Inspect a cookie and find out to which controller/slot it belongs. */ 1218static 1219struct b43_dmaring *parse_cookie(struct b43_wldev *dev, u16 cookie, int *slot) 1220{ 1221 struct b43_dma *dma = &dev->dma; 1222 struct b43_dmaring *ring = NULL; 1223 1224 switch (cookie & 0xF000) { 1225 case 0x1000: 1226 ring = dma->tx_ring_AC_BK; 1227 break; 1228 case 0x2000: 1229 ring = dma->tx_ring_AC_BE; 1230 break; 1231 case 0x3000: 1232 ring = dma->tx_ring_AC_VI; 1233 break; 1234 case 0x4000: 1235 ring = dma->tx_ring_AC_VO; 1236 break; 1237 case 0x5000: 1238 ring = dma->tx_ring_mcast; 1239 break; 1240 } 1241 *slot = (cookie & 0x0FFF); 1242 if (unlikely(!ring || *slot < 0 || *slot >= ring->nr_slots)) { 1243 b43dbg(dev->wl, "TX-status contains " 1244 "invalid cookie: 0x%04X\n", cookie); 1245 return NULL; 1246 } 1247 1248 return ring; 1249} 1250 1251static int dma_tx_fragment(struct b43_dmaring *ring, 1252 struct sk_buff *skb) 1253{ 1254 const struct b43_dma_ops *ops = ring->ops; 1255 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 1256 struct b43_private_tx_info *priv_info = b43_get_priv_tx_info(info); 1257 u8 *header; 1258 int slot, old_top_slot, old_used_slots; 1259 int err; 1260 struct b43_dmadesc_generic *desc; 1261 struct b43_dmadesc_meta *meta; 1262 struct b43_dmadesc_meta *meta_hdr; 1263 u16 cookie; 1264 size_t hdrsize = b43_txhdr_size(ring->dev); 1265 1266 /* Important note: If the number of used DMA slots per TX frame 1267 * is changed here, the TX_SLOTS_PER_FRAME definition at the top of 1268 * the file has to be updated, too! 1269 */ 1270 1271 old_top_slot = ring->current_slot; 1272 old_used_slots = ring->used_slots; 1273 1274 /* Get a slot for the header. */ 1275 slot = request_slot(ring); 1276 desc = ops->idx2desc(ring, slot, &meta_hdr); 1277 memset(meta_hdr, 0, sizeof(*meta_hdr)); 1278 1279 header = &(ring->txhdr_cache[(slot / TX_SLOTS_PER_FRAME) * hdrsize]); 1280 cookie = generate_cookie(ring, slot); 1281 err = b43_generate_txhdr(ring->dev, header, 1282 skb, info, cookie); 1283 if (unlikely(err)) { 1284 ring->current_slot = old_top_slot; 1285 ring->used_slots = old_used_slots; 1286 return err; 1287 } 1288 1289 meta_hdr->dmaaddr = map_descbuffer(ring, (unsigned char *)header, 1290 hdrsize, 1); 1291 if (b43_dma_mapping_error(ring, meta_hdr->dmaaddr, hdrsize, 1)) { 1292 ring->current_slot = old_top_slot; 1293 ring->used_slots = old_used_slots; 1294 return -EIO; 1295 } 1296 ops->fill_descriptor(ring, desc, meta_hdr->dmaaddr, 1297 hdrsize, 1, 0, 0); 1298 1299 /* Get a slot for the payload. */ 1300 slot = request_slot(ring); 1301 desc = ops->idx2desc(ring, slot, &meta); 1302 memset(meta, 0, sizeof(*meta)); 1303 1304 meta->skb = skb; 1305 meta->is_last_fragment = true; 1306 priv_info->bouncebuffer = NULL; 1307 1308 meta->dmaaddr = map_descbuffer(ring, skb->data, skb->len, 1); 1309 /* create a bounce buffer in zone_dma on mapping failure. */ 1310 if (b43_dma_mapping_error(ring, meta->dmaaddr, skb->len, 1)) { 1311 priv_info->bouncebuffer = kmemdup(skb->data, skb->len, 1312 GFP_ATOMIC | GFP_DMA); 1313 if (!priv_info->bouncebuffer) { 1314 ring->current_slot = old_top_slot; 1315 ring->used_slots = old_used_slots; 1316 err = -ENOMEM; 1317 goto out_unmap_hdr; 1318 } 1319 1320 meta->dmaaddr = map_descbuffer(ring, priv_info->bouncebuffer, skb->len, 1); 1321 if (b43_dma_mapping_error(ring, meta->dmaaddr, skb->len, 1)) { 1322 kfree(priv_info->bouncebuffer); 1323 priv_info->bouncebuffer = NULL; 1324 ring->current_slot = old_top_slot; 1325 ring->used_slots = old_used_slots; 1326 err = -EIO; 1327 goto out_unmap_hdr; 1328 } 1329 } 1330 1331 ops->fill_descriptor(ring, desc, meta->dmaaddr, skb->len, 0, 1, 1); 1332 1333 if (info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) { 1334 /* Tell the firmware about the cookie of the last 1335 * mcast frame, so it can clear the more-data bit in it. */ 1336 b43_shm_write16(ring->dev, B43_SHM_SHARED, 1337 B43_SHM_SH_MCASTCOOKIE, cookie); 1338 } 1339 /* Now transfer the whole frame. */ 1340 wmb(); 1341 ops->poke_tx(ring, next_slot(ring, slot)); 1342 return 0; 1343 1344out_unmap_hdr: 1345 unmap_descbuffer(ring, meta_hdr->dmaaddr, 1346 hdrsize, 1); 1347 return err; 1348} 1349 1350static inline int should_inject_overflow(struct b43_dmaring *ring) 1351{ 1352#ifdef CONFIG_B43_DEBUG 1353 if (unlikely(b43_debug(ring->dev, B43_DBG_DMAOVERFLOW))) { 1354 /* Check if we should inject another ringbuffer overflow 1355 * to test handling of this situation in the stack. */ 1356 unsigned long next_overflow; 1357 1358 next_overflow = ring->last_injected_overflow + HZ; 1359 if (time_after(jiffies, next_overflow)) { 1360 ring->last_injected_overflow = jiffies; 1361 b43dbg(ring->dev->wl, 1362 "Injecting TX ring overflow on " 1363 "DMA controller %d\n", ring->index); 1364 return 1; 1365 } 1366 } 1367#endif /* CONFIG_B43_DEBUG */ 1368 return 0; 1369} 1370 1371/* Static mapping of mac80211's queues (priorities) to b43 DMA rings. */ 1372static struct b43_dmaring *select_ring_by_priority(struct b43_wldev *dev, 1373 u8 queue_prio) 1374{ 1375 struct b43_dmaring *ring; 1376 1377 if (dev->qos_enabled) { 1378 /* 0 = highest priority */ 1379 switch (queue_prio) { 1380 default: 1381 B43_WARN_ON(1); 1382 /* fallthrough */ 1383 case 0: 1384 ring = dev->dma.tx_ring_AC_VO; 1385 break; 1386 case 1: 1387 ring = dev->dma.tx_ring_AC_VI; 1388 break; 1389 case 2: 1390 ring = dev->dma.tx_ring_AC_BE; 1391 break; 1392 case 3: 1393 ring = dev->dma.tx_ring_AC_BK; 1394 break; 1395 } 1396 } else 1397 ring = dev->dma.tx_ring_AC_BE; 1398 1399 return ring; 1400} 1401 1402int b43_dma_tx(struct b43_wldev *dev, struct sk_buff *skb) 1403{ 1404 struct b43_dmaring *ring; 1405 struct ieee80211_hdr *hdr; 1406 int err = 0; 1407 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 1408 1409 hdr = (struct ieee80211_hdr *)skb->data; 1410 if (info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) { 1411 /* The multicast ring will be sent after the DTIM */ 1412 ring = dev->dma.tx_ring_mcast; 1413 /* Set the more-data bit. Ucode will clear it on 1414 * the last frame for us. */ 1415 hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_MOREDATA); 1416 } else { 1417 /* Decide by priority where to put this frame. */ 1418 ring = select_ring_by_priority( 1419 dev, skb_get_queue_mapping(skb)); 1420 } 1421 1422 B43_WARN_ON(!ring->tx); 1423 1424 if (unlikely(ring->stopped)) { 1425 /* We get here only because of a bug in mac80211. 1426 * Because of a race, one packet may be queued after 1427 * the queue is stopped, thus we got called when we shouldn't. 1428 * For now, just refuse the transmit. */ 1429 if (b43_debug(dev, B43_DBG_DMAVERBOSE)) 1430 b43err(dev->wl, "Packet after queue stopped\n"); 1431 err = -ENOSPC; 1432 goto out; 1433 } 1434 1435 if (unlikely(WARN_ON(free_slots(ring) < TX_SLOTS_PER_FRAME))) { 1436 /* If we get here, we have a real error with the queue 1437 * full, but queues not stopped. */ 1438 b43err(dev->wl, "DMA queue overflow\n"); 1439 err = -ENOSPC; 1440 goto out; 1441 } 1442 1443 /* Assign the queue number to the ring (if not already done before) 1444 * so TX status handling can use it. The queue to ring mapping is 1445 * static, so we don't need to store it per frame. */ 1446 ring->queue_prio = skb_get_queue_mapping(skb); 1447 1448 err = dma_tx_fragment(ring, skb); 1449 if (unlikely(err == -ENOKEY)) { 1450 /* Drop this packet, as we don't have the encryption key 1451 * anymore and must not transmit it unencrypted. */ 1452 ieee80211_free_txskb(dev->wl->hw, skb); 1453 err = 0; 1454 goto out; 1455 } 1456 if (unlikely(err)) { 1457 b43err(dev->wl, "DMA tx mapping failure\n"); 1458 goto out; 1459 } 1460 if ((free_slots(ring) < TX_SLOTS_PER_FRAME) || 1461 should_inject_overflow(ring)) { 1462 /* This TX ring is full. */ 1463 unsigned int skb_mapping = skb_get_queue_mapping(skb); 1464 ieee80211_stop_queue(dev->wl->hw, skb_mapping); 1465 dev->wl->tx_queue_stopped[skb_mapping] = 1; 1466 ring->stopped = true; 1467 if (b43_debug(dev, B43_DBG_DMAVERBOSE)) { 1468 b43dbg(dev->wl, "Stopped TX ring %d\n", ring->index); 1469 } 1470 } 1471out: 1472 1473 return err; 1474} 1475 1476void b43_dma_handle_txstatus(struct b43_wldev *dev, 1477 const struct b43_txstatus *status) 1478{ 1479 const struct b43_dma_ops *ops; 1480 struct b43_dmaring *ring; 1481 struct b43_dmadesc_meta *meta; 1482 static const struct b43_txstatus fake; /* filled with 0 */ 1483 const struct b43_txstatus *txstat; 1484 int slot, firstused; 1485 bool frame_succeed; 1486 int skip; 1487 static u8 err_out1, err_out2; 1488 1489 ring = parse_cookie(dev, status->cookie, &slot); 1490 if (unlikely(!ring)) 1491 return; 1492 B43_WARN_ON(!ring->tx); 1493 1494 /* Sanity check: TX packets are processed in-order on one ring. 1495 * Check if the slot deduced from the cookie really is the first 1496 * used slot. */ 1497 firstused = ring->current_slot - ring->used_slots + 1; 1498 if (firstused < 0) 1499 firstused = ring->nr_slots + firstused; 1500 1501 skip = 0; 1502 if (unlikely(slot != firstused)) { 1503 /* This possibly is a firmware bug and will result in 1504 * malfunction, memory leaks and/or stall of DMA functionality. 1505 */ 1506 if (slot == next_slot(ring, next_slot(ring, firstused))) { 1507 /* If a single header/data pair was missed, skip over 1508 * the first two slots in an attempt to recover. 1509 */ 1510 slot = firstused; 1511 skip = 2; 1512 if (!err_out1) { 1513 /* Report the error once. */ 1514 b43dbg(dev->wl, 1515 "Skip on DMA ring %d slot %d.\n", 1516 ring->index, slot); 1517 err_out1 = 1; 1518 } 1519 } else { 1520 /* More than a single header/data pair were missed. 1521 * Report this error once. 1522 */ 1523 if (!err_out2) 1524 b43dbg(dev->wl, 1525 "Out of order TX status report on DMA ring %d. Expected %d, but got %d\n", 1526 ring->index, firstused, slot); 1527 err_out2 = 1; 1528 return; 1529 } 1530 } 1531 1532 ops = ring->ops; 1533 while (1) { 1534 B43_WARN_ON(slot < 0 || slot >= ring->nr_slots); 1535 /* get meta - ignore returned value */ 1536 ops->idx2desc(ring, slot, &meta); 1537 1538 if (b43_dma_ptr_is_poisoned(meta->skb)) { 1539 b43dbg(dev->wl, "Poisoned TX slot %d (first=%d) " 1540 "on ring %d\n", 1541 slot, firstused, ring->index); 1542 break; 1543 } 1544 1545 if (meta->skb) { 1546 struct b43_private_tx_info *priv_info = 1547 b43_get_priv_tx_info(IEEE80211_SKB_CB(meta->skb)); 1548 1549 unmap_descbuffer(ring, meta->dmaaddr, 1550 meta->skb->len, 1); 1551 kfree(priv_info->bouncebuffer); 1552 priv_info->bouncebuffer = NULL; 1553 } else { 1554 unmap_descbuffer(ring, meta->dmaaddr, 1555 b43_txhdr_size(dev), 1); 1556 } 1557 1558 if (meta->is_last_fragment) { 1559 struct ieee80211_tx_info *info; 1560 1561 if (unlikely(!meta->skb)) { 1562 /* This is a scatter-gather fragment of a frame, 1563 * so the skb pointer must not be NULL. 1564 */ 1565 b43dbg(dev->wl, "TX status unexpected NULL skb " 1566 "at slot %d (first=%d) on ring %d\n", 1567 slot, firstused, ring->index); 1568 break; 1569 } 1570 1571 info = IEEE80211_SKB_CB(meta->skb); 1572 1573 /* 1574 * Call back to inform the ieee80211 subsystem about 1575 * the status of the transmission. When skipping over 1576 * a missed TX status report, use a status structure 1577 * filled with zeros to indicate that the frame was not 1578 * sent (frame_count 0) and not acknowledged 1579 */ 1580 if (unlikely(skip)) 1581 txstat = &fake; 1582 else 1583 txstat = status; 1584 1585 frame_succeed = b43_fill_txstatus_report(dev, info, 1586 txstat); 1587#ifdef CONFIG_B43_DEBUG 1588 if (frame_succeed) 1589 ring->nr_succeed_tx_packets++; 1590 else 1591 ring->nr_failed_tx_packets++; 1592 ring->nr_total_packet_tries += status->frame_count; 1593#endif /* DEBUG */ 1594 ieee80211_tx_status(dev->wl->hw, meta->skb); 1595 1596 /* skb will be freed by ieee80211_tx_status(). 1597 * Poison our pointer. */ 1598 meta->skb = B43_DMA_PTR_POISON; 1599 } else { 1600 /* No need to call free_descriptor_buffer here, as 1601 * this is only the txhdr, which is not allocated. 1602 */ 1603 if (unlikely(meta->skb)) { 1604 b43dbg(dev->wl, "TX status unexpected non-NULL skb " 1605 "at slot %d (first=%d) on ring %d\n", 1606 slot, firstused, ring->index); 1607 break; 1608 } 1609 } 1610 1611 /* Everything unmapped and free'd. So it's not used anymore. */ 1612 ring->used_slots--; 1613 1614 if (meta->is_last_fragment && !skip) { 1615 /* This is the last scatter-gather 1616 * fragment of the frame. We are done. */ 1617 break; 1618 } 1619 slot = next_slot(ring, slot); 1620 if (skip > 0) 1621 --skip; 1622 } 1623 if (ring->stopped) { 1624 B43_WARN_ON(free_slots(ring) < TX_SLOTS_PER_FRAME); 1625 ring->stopped = false; 1626 } 1627 1628 if (dev->wl->tx_queue_stopped[ring->queue_prio]) { 1629 dev->wl->tx_queue_stopped[ring->queue_prio] = 0; 1630 } else { 1631 /* If the driver queue is running wake the corresponding 1632 * mac80211 queue. */ 1633 ieee80211_wake_queue(dev->wl->hw, ring->queue_prio); 1634 if (b43_debug(dev, B43_DBG_DMAVERBOSE)) { 1635 b43dbg(dev->wl, "Woke up TX ring %d\n", ring->index); 1636 } 1637 } 1638 /* Add work to the queue. */ 1639 ieee80211_queue_work(dev->wl->hw, &dev->wl->tx_work); 1640} 1641 1642static void dma_rx(struct b43_dmaring *ring, int *slot) 1643{ 1644 const struct b43_dma_ops *ops = ring->ops; 1645 struct b43_dmadesc_generic *desc; 1646 struct b43_dmadesc_meta *meta; 1647 struct b43_rxhdr_fw4 *rxhdr; 1648 struct sk_buff *skb; 1649 u16 len; 1650 int err; 1651 dma_addr_t dmaaddr; 1652 1653 desc = ops->idx2desc(ring, *slot, &meta); 1654 1655 sync_descbuffer_for_cpu(ring, meta->dmaaddr, ring->rx_buffersize); 1656 skb = meta->skb; 1657 1658 rxhdr = (struct b43_rxhdr_fw4 *)skb->data; 1659 len = le16_to_cpu(rxhdr->frame_len); 1660 if (len == 0) { 1661 int i = 0; 1662 1663 do { 1664 udelay(2); 1665 barrier(); 1666 len = le16_to_cpu(rxhdr->frame_len); 1667 } while (len == 0 && i++ < 5); 1668 if (unlikely(len == 0)) { 1669 dmaaddr = meta->dmaaddr; 1670 goto drop_recycle_buffer; 1671 } 1672 } 1673 if (unlikely(b43_rx_buffer_is_poisoned(ring, skb))) { 1674 /* Something went wrong with the DMA. 1675 * The device did not touch the buffer and did not overwrite the poison. */ 1676 b43dbg(ring->dev->wl, "DMA RX: Dropping poisoned buffer.\n"); 1677 dmaaddr = meta->dmaaddr; 1678 goto drop_recycle_buffer; 1679 } 1680 if (unlikely(len + ring->frameoffset > ring->rx_buffersize)) { 1681 /* The data did not fit into one descriptor buffer 1682 * and is split over multiple buffers. 1683 * This should never happen, as we try to allocate buffers 1684 * big enough. So simply ignore this packet. 1685 */ 1686 int cnt = 0; 1687 s32 tmp = len; 1688 1689 while (1) { 1690 desc = ops->idx2desc(ring, *slot, &meta); 1691 /* recycle the descriptor buffer. */ 1692 b43_poison_rx_buffer(ring, meta->skb); 1693 sync_descbuffer_for_device(ring, meta->dmaaddr, 1694 ring->rx_buffersize); 1695 *slot = next_slot(ring, *slot); 1696 cnt++; 1697 tmp -= ring->rx_buffersize; 1698 if (tmp <= 0) 1699 break; 1700 } 1701 b43err(ring->dev->wl, "DMA RX buffer too small " 1702 "(len: %u, buffer: %u, nr-dropped: %d)\n", 1703 len, ring->rx_buffersize, cnt); 1704 goto drop; 1705 } 1706 1707 dmaaddr = meta->dmaaddr; 1708 err = setup_rx_descbuffer(ring, desc, meta, GFP_ATOMIC); 1709 if (unlikely(err)) { 1710 b43dbg(ring->dev->wl, "DMA RX: setup_rx_descbuffer() failed\n"); 1711 goto drop_recycle_buffer; 1712 } 1713 1714 unmap_descbuffer(ring, dmaaddr, ring->rx_buffersize, 0); 1715 skb_put(skb, len + ring->frameoffset); 1716 skb_pull(skb, ring->frameoffset); 1717 1718 b43_rx(ring->dev, skb, rxhdr); 1719drop: 1720 return; 1721 1722drop_recycle_buffer: 1723 /* Poison and recycle the RX buffer. */ 1724 b43_poison_rx_buffer(ring, skb); 1725 sync_descbuffer_for_device(ring, dmaaddr, ring->rx_buffersize); 1726} 1727 1728void b43_dma_handle_rx_overflow(struct b43_dmaring *ring) 1729{ 1730 int current_slot, previous_slot; 1731 1732 B43_WARN_ON(ring->tx); 1733 1734 /* Device has filled all buffers, drop all packets and let TCP 1735 * decrease speed. 1736 * Decrement RX index by one will let the device to see all slots 1737 * as free again 1738 */ 1739 /* 1740 *TODO: How to increase rx_drop in mac80211? 1741 */ 1742 current_slot = ring->ops->get_current_rxslot(ring); 1743 previous_slot = prev_slot(ring, current_slot); 1744 ring->ops->set_current_rxslot(ring, previous_slot); 1745} 1746 1747void b43_dma_rx(struct b43_dmaring *ring) 1748{ 1749 const struct b43_dma_ops *ops = ring->ops; 1750 int slot, current_slot; 1751 int used_slots = 0; 1752 1753 B43_WARN_ON(ring->tx); 1754 current_slot = ops->get_current_rxslot(ring); 1755 B43_WARN_ON(!(current_slot >= 0 && current_slot < ring->nr_slots)); 1756 1757 slot = ring->current_slot; 1758 for (; slot != current_slot; slot = next_slot(ring, slot)) { 1759 dma_rx(ring, &slot); 1760 update_max_used_slots(ring, ++used_slots); 1761 } 1762 wmb(); 1763 ops->set_current_rxslot(ring, slot); 1764 ring->current_slot = slot; 1765} 1766 1767static void b43_dma_tx_suspend_ring(struct b43_dmaring *ring) 1768{ 1769 B43_WARN_ON(!ring->tx); 1770 ring->ops->tx_suspend(ring); 1771} 1772 1773static void b43_dma_tx_resume_ring(struct b43_dmaring *ring) 1774{ 1775 B43_WARN_ON(!ring->tx); 1776 ring->ops->tx_resume(ring); 1777} 1778 1779void b43_dma_tx_suspend(struct b43_wldev *dev) 1780{ 1781 b43_power_saving_ctl_bits(dev, B43_PS_AWAKE); 1782 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_BK); 1783 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_BE); 1784 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_VI); 1785 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_VO); 1786 b43_dma_tx_suspend_ring(dev->dma.tx_ring_mcast); 1787} 1788 1789void b43_dma_tx_resume(struct b43_wldev *dev) 1790{ 1791 b43_dma_tx_resume_ring(dev->dma.tx_ring_mcast); 1792 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_VO); 1793 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_VI); 1794 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_BE); 1795 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_BK); 1796 b43_power_saving_ctl_bits(dev, 0); 1797} 1798 1799static void direct_fifo_rx(struct b43_wldev *dev, enum b43_dmatype type, 1800 u16 mmio_base, bool enable) 1801{ 1802 u32 ctl; 1803 1804 if (type == B43_DMA_64BIT) { 1805 ctl = b43_read32(dev, mmio_base + B43_DMA64_RXCTL); 1806 ctl &= ~B43_DMA64_RXDIRECTFIFO; 1807 if (enable) 1808 ctl |= B43_DMA64_RXDIRECTFIFO; 1809 b43_write32(dev, mmio_base + B43_DMA64_RXCTL, ctl); 1810 } else { 1811 ctl = b43_read32(dev, mmio_base + B43_DMA32_RXCTL); 1812 ctl &= ~B43_DMA32_RXDIRECTFIFO; 1813 if (enable) 1814 ctl |= B43_DMA32_RXDIRECTFIFO; 1815 b43_write32(dev, mmio_base + B43_DMA32_RXCTL, ctl); 1816 } 1817} 1818 1819/* Enable/Disable Direct FIFO Receive Mode (PIO) on a RX engine. 1820 * This is called from PIO code, so DMA structures are not available. */ 1821void b43_dma_direct_fifo_rx(struct b43_wldev *dev, 1822 unsigned int engine_index, bool enable) 1823{ 1824 enum b43_dmatype type; 1825 u16 mmio_base; 1826 1827 type = dma_mask_to_engine_type(supported_dma_mask(dev)); 1828 1829 mmio_base = b43_dmacontroller_base(type, engine_index); 1830 direct_fifo_rx(dev, type, mmio_base, enable); 1831} 1832