1/* 2 * DMA Pool allocator 3 * 4 * Copyright 2001 David Brownell 5 * Copyright 2007 Intel Corporation 6 * Author: Matthew Wilcox <willy@linux.intel.com> 7 * 8 * This software may be redistributed and/or modified under the terms of 9 * the GNU General Public License ("GPL") version 2 as published by the 10 * Free Software Foundation. 11 * 12 * This allocator returns small blocks of a given size which are DMA-able by 13 * the given device. It uses the dma_alloc_coherent page allocator to get 14 * new pages, then splits them up into blocks of the required size. 15 * Many older drivers still have their own code to do this. 16 * 17 * The current design of this allocator is fairly simple. The pool is 18 * represented by the 'struct dma_pool' which keeps a doubly-linked list of 19 * allocated pages. Each page in the page_list is split into blocks of at 20 * least 'size' bytes. Free blocks are tracked in an unsorted singly-linked 21 * list of free blocks within the page. Used blocks aren't tracked, but we 22 * keep a count of how many are currently allocated from each page. 23 */ 24 25#include <linux/device.h> 26#include <linux/dma-mapping.h> 27#include <linux/dmapool.h> 28#include <linux/kernel.h> 29#include <linux/list.h> 30#include <linux/export.h> 31#include <linux/mutex.h> 32#include <linux/poison.h> 33#include <linux/sched.h> 34#include <linux/slab.h> 35#include <linux/stat.h> 36#include <linux/spinlock.h> 37#include <linux/string.h> 38#include <linux/types.h> 39#include <linux/wait.h> 40 41#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON) 42#define DMAPOOL_DEBUG 1 43#endif 44 45struct dma_pool { /* the pool */ 46 struct list_head page_list; 47 spinlock_t lock; 48 size_t size; 49 struct device *dev; 50 size_t allocation; 51 size_t boundary; 52 char name[32]; 53 struct list_head pools; 54}; 55 56struct dma_page { /* cacheable header for 'allocation' bytes */ 57 struct list_head page_list; 58 void *vaddr; 59 dma_addr_t dma; 60 unsigned int in_use; 61 unsigned int offset; 62}; 63 64static DEFINE_MUTEX(pools_lock); 65static DEFINE_MUTEX(pools_reg_lock); 66 67static ssize_t 68show_pools(struct device *dev, struct device_attribute *attr, char *buf) 69{ 70 unsigned temp; 71 unsigned size; 72 char *next; 73 struct dma_page *page; 74 struct dma_pool *pool; 75 76 next = buf; 77 size = PAGE_SIZE; 78 79 temp = scnprintf(next, size, "poolinfo - 0.1\n"); 80 size -= temp; 81 next += temp; 82 83 mutex_lock(&pools_lock); 84 list_for_each_entry(pool, &dev->dma_pools, pools) { 85 unsigned pages = 0; 86 unsigned blocks = 0; 87 88 spin_lock_irq(&pool->lock); 89 list_for_each_entry(page, &pool->page_list, page_list) { 90 pages++; 91 blocks += page->in_use; 92 } 93 spin_unlock_irq(&pool->lock); 94 95 /* per-pool info, no real statistics yet */ 96 temp = scnprintf(next, size, "%-16s %4u %4Zu %4Zu %2u\n", 97 pool->name, blocks, 98 pages * (pool->allocation / pool->size), 99 pool->size, pages); 100 size -= temp; 101 next += temp; 102 } 103 mutex_unlock(&pools_lock); 104 105 return PAGE_SIZE - size; 106} 107 108static DEVICE_ATTR(pools, S_IRUGO, show_pools, NULL); 109 110/** 111 * dma_pool_create - Creates a pool of consistent memory blocks, for dma. 112 * @name: name of pool, for diagnostics 113 * @dev: device that will be doing the DMA 114 * @size: size of the blocks in this pool. 115 * @align: alignment requirement for blocks; must be a power of two 116 * @boundary: returned blocks won't cross this power of two boundary 117 * Context: !in_interrupt() 118 * 119 * Returns a dma allocation pool with the requested characteristics, or 120 * null if one can't be created. Given one of these pools, dma_pool_alloc() 121 * may be used to allocate memory. Such memory will all have "consistent" 122 * DMA mappings, accessible by the device and its driver without using 123 * cache flushing primitives. The actual size of blocks allocated may be 124 * larger than requested because of alignment. 125 * 126 * If @boundary is nonzero, objects returned from dma_pool_alloc() won't 127 * cross that size boundary. This is useful for devices which have 128 * addressing restrictions on individual DMA transfers, such as not crossing 129 * boundaries of 4KBytes. 130 */ 131struct dma_pool *dma_pool_create(const char *name, struct device *dev, 132 size_t size, size_t align, size_t boundary) 133{ 134 struct dma_pool *retval; 135 size_t allocation; 136 bool empty = false; 137 138 if (align == 0) 139 align = 1; 140 else if (align & (align - 1)) 141 return NULL; 142 143 if (size == 0) 144 return NULL; 145 else if (size < 4) 146 size = 4; 147 148 if ((size % align) != 0) 149 size = ALIGN(size, align); 150 151 allocation = max_t(size_t, size, PAGE_SIZE); 152 153 if (!boundary) 154 boundary = allocation; 155 else if ((boundary < size) || (boundary & (boundary - 1))) 156 return NULL; 157 158 retval = kmalloc_node(sizeof(*retval), GFP_KERNEL, dev_to_node(dev)); 159 if (!retval) 160 return retval; 161 162 strlcpy(retval->name, name, sizeof(retval->name)); 163 164 retval->dev = dev; 165 166 INIT_LIST_HEAD(&retval->page_list); 167 spin_lock_init(&retval->lock); 168 retval->size = size; 169 retval->boundary = boundary; 170 retval->allocation = allocation; 171 172 INIT_LIST_HEAD(&retval->pools); 173 174 /* 175 * pools_lock ensures that the ->dma_pools list does not get corrupted. 176 * pools_reg_lock ensures that there is not a race between 177 * dma_pool_create() and dma_pool_destroy() or within dma_pool_create() 178 * when the first invocation of dma_pool_create() failed on 179 * device_create_file() and the second assumes that it has been done (I 180 * know it is a short window). 181 */ 182 mutex_lock(&pools_reg_lock); 183 mutex_lock(&pools_lock); 184 if (list_empty(&dev->dma_pools)) 185 empty = true; 186 list_add(&retval->pools, &dev->dma_pools); 187 mutex_unlock(&pools_lock); 188 if (empty) { 189 int err; 190 191 err = device_create_file(dev, &dev_attr_pools); 192 if (err) { 193 mutex_lock(&pools_lock); 194 list_del(&retval->pools); 195 mutex_unlock(&pools_lock); 196 mutex_unlock(&pools_reg_lock); 197 kfree(retval); 198 return NULL; 199 } 200 } 201 mutex_unlock(&pools_reg_lock); 202 return retval; 203} 204EXPORT_SYMBOL(dma_pool_create); 205 206static void pool_initialise_page(struct dma_pool *pool, struct dma_page *page) 207{ 208 unsigned int offset = 0; 209 unsigned int next_boundary = pool->boundary; 210 211 do { 212 unsigned int next = offset + pool->size; 213 if (unlikely((next + pool->size) >= next_boundary)) { 214 next = next_boundary; 215 next_boundary += pool->boundary; 216 } 217 *(int *)(page->vaddr + offset) = next; 218 offset = next; 219 } while (offset < pool->allocation); 220} 221 222static struct dma_page *pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags) 223{ 224 struct dma_page *page; 225 226 page = kmalloc(sizeof(*page), mem_flags); 227 if (!page) 228 return NULL; 229 page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation, 230 &page->dma, mem_flags); 231 if (page->vaddr) { 232#ifdef DMAPOOL_DEBUG 233 memset(page->vaddr, POOL_POISON_FREED, pool->allocation); 234#endif 235 pool_initialise_page(pool, page); 236 page->in_use = 0; 237 page->offset = 0; 238 } else { 239 kfree(page); 240 page = NULL; 241 } 242 return page; 243} 244 245static inline int is_page_busy(struct dma_page *page) 246{ 247 return page->in_use != 0; 248} 249 250static void pool_free_page(struct dma_pool *pool, struct dma_page *page) 251{ 252 dma_addr_t dma = page->dma; 253 254#ifdef DMAPOOL_DEBUG 255 memset(page->vaddr, POOL_POISON_FREED, pool->allocation); 256#endif 257 dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma); 258 list_del(&page->page_list); 259 kfree(page); 260} 261 262/** 263 * dma_pool_destroy - destroys a pool of dma memory blocks. 264 * @pool: dma pool that will be destroyed 265 * Context: !in_interrupt() 266 * 267 * Caller guarantees that no more memory from the pool is in use, 268 * and that nothing will try to use the pool after this call. 269 */ 270void dma_pool_destroy(struct dma_pool *pool) 271{ 272 bool empty = false; 273 274 mutex_lock(&pools_reg_lock); 275 mutex_lock(&pools_lock); 276 list_del(&pool->pools); 277 if (pool->dev && list_empty(&pool->dev->dma_pools)) 278 empty = true; 279 mutex_unlock(&pools_lock); 280 if (empty) 281 device_remove_file(pool->dev, &dev_attr_pools); 282 mutex_unlock(&pools_reg_lock); 283 284 while (!list_empty(&pool->page_list)) { 285 struct dma_page *page; 286 page = list_entry(pool->page_list.next, 287 struct dma_page, page_list); 288 if (is_page_busy(page)) { 289 if (pool->dev) 290 dev_err(pool->dev, 291 "dma_pool_destroy %s, %p busy\n", 292 pool->name, page->vaddr); 293 else 294 printk(KERN_ERR 295 "dma_pool_destroy %s, %p busy\n", 296 pool->name, page->vaddr); 297 /* leak the still-in-use consistent memory */ 298 list_del(&page->page_list); 299 kfree(page); 300 } else 301 pool_free_page(pool, page); 302 } 303 304 kfree(pool); 305} 306EXPORT_SYMBOL(dma_pool_destroy); 307 308/** 309 * dma_pool_alloc - get a block of consistent memory 310 * @pool: dma pool that will produce the block 311 * @mem_flags: GFP_* bitmask 312 * @handle: pointer to dma address of block 313 * 314 * This returns the kernel virtual address of a currently unused block, 315 * and reports its dma address through the handle. 316 * If such a memory block can't be allocated, %NULL is returned. 317 */ 318void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, 319 dma_addr_t *handle) 320{ 321 unsigned long flags; 322 struct dma_page *page; 323 size_t offset; 324 void *retval; 325 326 might_sleep_if(mem_flags & __GFP_WAIT); 327 328 spin_lock_irqsave(&pool->lock, flags); 329 list_for_each_entry(page, &pool->page_list, page_list) { 330 if (page->offset < pool->allocation) 331 goto ready; 332 } 333 334 /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */ 335 spin_unlock_irqrestore(&pool->lock, flags); 336 337 page = pool_alloc_page(pool, mem_flags); 338 if (!page) 339 return NULL; 340 341 spin_lock_irqsave(&pool->lock, flags); 342 343 list_add(&page->page_list, &pool->page_list); 344 ready: 345 page->in_use++; 346 offset = page->offset; 347 page->offset = *(int *)(page->vaddr + offset); 348 retval = offset + page->vaddr; 349 *handle = offset + page->dma; 350#ifdef DMAPOOL_DEBUG 351 { 352 int i; 353 u8 *data = retval; 354 /* page->offset is stored in first 4 bytes */ 355 for (i = sizeof(page->offset); i < pool->size; i++) { 356 if (data[i] == POOL_POISON_FREED) 357 continue; 358 if (pool->dev) 359 dev_err(pool->dev, 360 "dma_pool_alloc %s, %p (corrupted)\n", 361 pool->name, retval); 362 else 363 pr_err("dma_pool_alloc %s, %p (corrupted)\n", 364 pool->name, retval); 365 366 /* 367 * Dump the first 4 bytes even if they are not 368 * POOL_POISON_FREED 369 */ 370 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, 371 data, pool->size, 1); 372 break; 373 } 374 } 375 memset(retval, POOL_POISON_ALLOCATED, pool->size); 376#endif 377 spin_unlock_irqrestore(&pool->lock, flags); 378 return retval; 379} 380EXPORT_SYMBOL(dma_pool_alloc); 381 382static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t dma) 383{ 384 struct dma_page *page; 385 386 list_for_each_entry(page, &pool->page_list, page_list) { 387 if (dma < page->dma) 388 continue; 389 if (dma < (page->dma + pool->allocation)) 390 return page; 391 } 392 return NULL; 393} 394 395/** 396 * dma_pool_free - put block back into dma pool 397 * @pool: the dma pool holding the block 398 * @vaddr: virtual address of block 399 * @dma: dma address of block 400 * 401 * Caller promises neither device nor driver will again touch this block 402 * unless it is first re-allocated. 403 */ 404void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma) 405{ 406 struct dma_page *page; 407 unsigned long flags; 408 unsigned int offset; 409 410 spin_lock_irqsave(&pool->lock, flags); 411 page = pool_find_page(pool, dma); 412 if (!page) { 413 spin_unlock_irqrestore(&pool->lock, flags); 414 if (pool->dev) 415 dev_err(pool->dev, 416 "dma_pool_free %s, %p/%lx (bad dma)\n", 417 pool->name, vaddr, (unsigned long)dma); 418 else 419 printk(KERN_ERR "dma_pool_free %s, %p/%lx (bad dma)\n", 420 pool->name, vaddr, (unsigned long)dma); 421 return; 422 } 423 424 offset = vaddr - page->vaddr; 425#ifdef DMAPOOL_DEBUG 426 if ((dma - page->dma) != offset) { 427 spin_unlock_irqrestore(&pool->lock, flags); 428 if (pool->dev) 429 dev_err(pool->dev, 430 "dma_pool_free %s, %p (bad vaddr)/%Lx\n", 431 pool->name, vaddr, (unsigned long long)dma); 432 else 433 printk(KERN_ERR 434 "dma_pool_free %s, %p (bad vaddr)/%Lx\n", 435 pool->name, vaddr, (unsigned long long)dma); 436 return; 437 } 438 { 439 unsigned int chain = page->offset; 440 while (chain < pool->allocation) { 441 if (chain != offset) { 442 chain = *(int *)(page->vaddr + chain); 443 continue; 444 } 445 spin_unlock_irqrestore(&pool->lock, flags); 446 if (pool->dev) 447 dev_err(pool->dev, "dma_pool_free %s, dma %Lx " 448 "already free\n", pool->name, 449 (unsigned long long)dma); 450 else 451 printk(KERN_ERR "dma_pool_free %s, dma %Lx " 452 "already free\n", pool->name, 453 (unsigned long long)dma); 454 return; 455 } 456 } 457 memset(vaddr, POOL_POISON_FREED, pool->size); 458#endif 459 460 page->in_use--; 461 *(int *)vaddr = page->offset; 462 page->offset = offset; 463 /* 464 * Resist a temptation to do 465 * if (!is_page_busy(page)) pool_free_page(pool, page); 466 * Better have a few empty pages hang around. 467 */ 468 spin_unlock_irqrestore(&pool->lock, flags); 469} 470EXPORT_SYMBOL(dma_pool_free); 471 472/* 473 * Managed DMA pool 474 */ 475static void dmam_pool_release(struct device *dev, void *res) 476{ 477 struct dma_pool *pool = *(struct dma_pool **)res; 478 479 dma_pool_destroy(pool); 480} 481 482static int dmam_pool_match(struct device *dev, void *res, void *match_data) 483{ 484 return *(struct dma_pool **)res == match_data; 485} 486 487/** 488 * dmam_pool_create - Managed dma_pool_create() 489 * @name: name of pool, for diagnostics 490 * @dev: device that will be doing the DMA 491 * @size: size of the blocks in this pool. 492 * @align: alignment requirement for blocks; must be a power of two 493 * @allocation: returned blocks won't cross this boundary (or zero) 494 * 495 * Managed dma_pool_create(). DMA pool created with this function is 496 * automatically destroyed on driver detach. 497 */ 498struct dma_pool *dmam_pool_create(const char *name, struct device *dev, 499 size_t size, size_t align, size_t allocation) 500{ 501 struct dma_pool **ptr, *pool; 502 503 ptr = devres_alloc(dmam_pool_release, sizeof(*ptr), GFP_KERNEL); 504 if (!ptr) 505 return NULL; 506 507 pool = *ptr = dma_pool_create(name, dev, size, align, allocation); 508 if (pool) 509 devres_add(dev, ptr); 510 else 511 devres_free(ptr); 512 513 return pool; 514} 515EXPORT_SYMBOL(dmam_pool_create); 516 517/** 518 * dmam_pool_destroy - Managed dma_pool_destroy() 519 * @pool: dma pool that will be destroyed 520 * 521 * Managed dma_pool_destroy(). 522 */ 523void dmam_pool_destroy(struct dma_pool *pool) 524{ 525 struct device *dev = pool->dev; 526 527 WARN_ON(devres_release(dev, dmam_pool_release, dmam_pool_match, pool)); 528} 529EXPORT_SYMBOL(dmam_pool_destroy); 530