root/drivers/gpu/drm/gma500/mmu.c

DEFINITIONS

1. psb_mmu_pt_index
2. psb_mmu_pd_index
3. psb_clflush
4. psb_mmu_clflush
5. psb_mmu_clflush
6. psb_mmu_flush_pd_locked
7. psb_mmu_flush_pd
8. psb_mmu_flush
9. psb_mmu_set_pd_context
10. psb_pd_addr_end
11. psb_mmu_mask_pte
12. psb_mmu_alloc_pd
13. psb_mmu_free_pt
14. psb_mmu_free_pagedir
15. psb_mmu_alloc_pt
16. psb_mmu_pt_alloc_map_lock
17. psb_mmu_pt_map_lock
18. psb_mmu_pt_unmap_unlock
19. psb_mmu_set_pte
20. psb_mmu_invalidate_pte
21. psb_mmu_get_default_pd
22. psb_get_default_pd_addr
23. psb_mmu_driver_takedown
24. psb_mmu_driver_init
25. psb_mmu_flush_ptes
26. psb_mmu_flush_ptes
27. psb_mmu_remove_pfn_sequence
28. psb_mmu_remove_pages
29. psb_mmu_insert_pfn_sequence
30. psb_mmu_insert_pages
31. psb_mmu_virtual_to_pfn

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**************************************************************************
   3  * Copyright (c) 2007, Intel Corporation.
   4  *
   5  **************************************************************************/
   6 
   7 #include <linux/highmem.h>
   8 
   9 #include "mmu.h"
  10 #include "psb_drv.h"
  11 #include "psb_reg.h"
  12 
  13 /*
  14  * Code for the SGX MMU:
  15  */
  16 
  17 /*
  18  * clflush on one processor only:
  19  * clflush should apparently flush the cache line on all processors in an
  20  * SMP system.
  21  */
  22 
  23 /*
  24  * kmap atomic:
  25  * The usage of the slots must be completely encapsulated within a spinlock, and
  26  * no other functions that may be using the locks for other purposed may be
  27  * called from within the locked region.
  28  * Since the slots are per processor, this will guarantee that we are the only
  29  * user.
  30  */
  31 
  32 /*
  33  * TODO: Inserting ptes from an interrupt handler:
  34  * This may be desirable for some SGX functionality where the GPU can fault in
  35  * needed pages. For that, we need to make an atomic insert_pages function, that
  36  * may fail.
  37  * If it fails, the caller need to insert the page using a workqueue function,
  38  * but on average it should be fast.
  39  */
  40 
  41 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
  42 {
  43         return (offset >> PSB_PTE_SHIFT) & 0x3FF;
  44 }
  45 
  46 static inline uint32_t psb_mmu_pd_index(uint32_t offset)
  47 {
  48         return offset >> PSB_PDE_SHIFT;
  49 }
  50 
  51 #if defined(CONFIG_X86)
  52 static inline void psb_clflush(void *addr)
  53 {
  54         __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
  55 }
  56 
  57 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
  58 {
  59         if (!driver->has_clflush)
  60                 return;
  61 
  62         mb();
  63         psb_clflush(addr);
  64         mb();
  65 }
  66 #else
  67 
  68 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
  69 {;
  70 }
  71 
  72 #endif
  73 
  74 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
  75 {
  76         struct drm_device *dev = driver->dev;
  77         struct drm_psb_private *dev_priv = dev->dev_private;
  78 
  79         if (atomic_read(&driver->needs_tlbflush) || force) {
  80                 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
  81                 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
  82 
  83                 /* Make sure data cache is turned off before enabling it */
  84                 wmb();
  85                 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
  86                 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
  87                 if (driver->msvdx_mmu_invaldc)
  88                         atomic_set(driver->msvdx_mmu_invaldc, 1);
  89         }
  90         atomic_set(&driver->needs_tlbflush, 0);
  91 }
  92 
  93 #if 0
  94 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
  95 {
  96         down_write(&driver->sem);
  97         psb_mmu_flush_pd_locked(driver, force);
  98         up_write(&driver->sem);
  99 }
 100 #endif
 101 
 102 void psb_mmu_flush(struct psb_mmu_driver *driver)
 103 {
 104         struct drm_device *dev = driver->dev;
 105         struct drm_psb_private *dev_priv = dev->dev_private;
 106         uint32_t val;
 107 
 108         down_write(&driver->sem);
 109         val = PSB_RSGX32(PSB_CR_BIF_CTRL);
 110         if (atomic_read(&driver->needs_tlbflush))
 111                 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
 112         else
 113                 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
 114 
 115         /* Make sure data cache is turned off and MMU is flushed before
 116            restoring bank interface control register */
 117         wmb();
 118         PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
 119                    PSB_CR_BIF_CTRL);
 120         (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
 121 
 122         atomic_set(&driver->needs_tlbflush, 0);
 123         if (driver->msvdx_mmu_invaldc)
 124                 atomic_set(driver->msvdx_mmu_invaldc, 1);
 125         up_write(&driver->sem);
 126 }
 127 
 128 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
 129 {
 130         struct drm_device *dev = pd->driver->dev;
 131         struct drm_psb_private *dev_priv = dev->dev_private;
 132         uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
 133                           PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
 134 
 135         down_write(&pd->driver->sem);
 136         PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
 137         wmb();
 138         psb_mmu_flush_pd_locked(pd->driver, 1);
 139         pd->hw_context = hw_context;
 140         up_write(&pd->driver->sem);
 141 
 142 }
 143 
 144 static inline unsigned long psb_pd_addr_end(unsigned long addr,
 145                                             unsigned long end)
 146 {
 147         addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
 148         return (addr < end) ? addr : end;
 149 }
 150 
 151 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
 152 {
 153         uint32_t mask = PSB_PTE_VALID;
 154 
 155         if (type & PSB_MMU_CACHED_MEMORY)
 156                 mask |= PSB_PTE_CACHED;
 157         if (type & PSB_MMU_RO_MEMORY)
 158                 mask |= PSB_PTE_RO;
 159         if (type & PSB_MMU_WO_MEMORY)
 160                 mask |= PSB_PTE_WO;
 161 
 162         return (pfn << PAGE_SHIFT) | mask;
 163 }
 164 
 165 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
 166                                     int trap_pagefaults, int invalid_type)
 167 {
 168         struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
 169         uint32_t *v;
 170         int i;
 171 
 172         if (!pd)
 173                 return NULL;
 174 
 175         pd->p = alloc_page(GFP_DMA32);
 176         if (!pd->p)
 177                 goto out_err1;
 178         pd->dummy_pt = alloc_page(GFP_DMA32);
 179         if (!pd->dummy_pt)
 180                 goto out_err2;
 181         pd->dummy_page = alloc_page(GFP_DMA32);
 182         if (!pd->dummy_page)
 183                 goto out_err3;
 184 
 185         if (!trap_pagefaults) {
 186                 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
 187                                                    invalid_type);
 188                 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
 189                                                    invalid_type);
 190         } else {
 191                 pd->invalid_pde = 0;
 192                 pd->invalid_pte = 0;
 193         }
 194 
 195         v = kmap(pd->dummy_pt);
 196         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 197                 v[i] = pd->invalid_pte;
 198 
 199         kunmap(pd->dummy_pt);
 200 
 201         v = kmap(pd->p);
 202         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 203                 v[i] = pd->invalid_pde;
 204 
 205         kunmap(pd->p);
 206 
 207         clear_page(kmap(pd->dummy_page));
 208         kunmap(pd->dummy_page);
 209 
 210         pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
 211         if (!pd->tables)
 212                 goto out_err4;
 213 
 214         pd->hw_context = -1;
 215         pd->pd_mask = PSB_PTE_VALID;
 216         pd->driver = driver;
 217 
 218         return pd;
 219 
 220 out_err4:
 221         __free_page(pd->dummy_page);
 222 out_err3:
 223         __free_page(pd->dummy_pt);
 224 out_err2:
 225         __free_page(pd->p);
 226 out_err1:
 227         kfree(pd);
 228         return NULL;
 229 }
 230 
 231 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
 232 {
 233         __free_page(pt->p);
 234         kfree(pt);
 235 }
 236 
 237 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
 238 {
 239         struct psb_mmu_driver *driver = pd->driver;
 240         struct drm_device *dev = driver->dev;
 241         struct drm_psb_private *dev_priv = dev->dev_private;
 242         struct psb_mmu_pt *pt;
 243         int i;
 244 
 245         down_write(&driver->sem);
 246         if (pd->hw_context != -1) {
 247                 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
 248                 psb_mmu_flush_pd_locked(driver, 1);
 249         }
 250 
 251         /* Should take the spinlock here, but we don't need to do that
 252            since we have the semaphore in write mode. */
 253 
 254         for (i = 0; i < 1024; ++i) {
 255                 pt = pd->tables[i];
 256                 if (pt)
 257                         psb_mmu_free_pt(pt);
 258         }
 259 
 260         vfree(pd->tables);
 261         __free_page(pd->dummy_page);
 262         __free_page(pd->dummy_pt);
 263         __free_page(pd->p);
 264         kfree(pd);
 265         up_write(&driver->sem);
 266 }
 267 
 268 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
 269 {
 270         struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
 271         void *v;
 272         uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
 273         uint32_t clflush_count = PAGE_SIZE / clflush_add;
 274         spinlock_t *lock = &pd->driver->lock;
 275         uint8_t *clf;
 276         uint32_t *ptes;
 277         int i;
 278 
 279         if (!pt)
 280                 return NULL;
 281 
 282         pt->p = alloc_page(GFP_DMA32);
 283         if (!pt->p) {
 284                 kfree(pt);
 285                 return NULL;
 286         }
 287 
 288         spin_lock(lock);
 289 
 290         v = kmap_atomic(pt->p);
 291         clf = (uint8_t *) v;
 292         ptes = (uint32_t *) v;
 293         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 294                 *ptes++ = pd->invalid_pte;
 295 
 296 #if defined(CONFIG_X86)
 297         if (pd->driver->has_clflush && pd->hw_context != -1) {
 298                 mb();
 299                 for (i = 0; i < clflush_count; ++i) {
 300                         psb_clflush(clf);
 301                         clf += clflush_add;
 302                 }
 303                 mb();
 304         }
 305 #endif
 306         kunmap_atomic(v);
 307         spin_unlock(lock);
 308 
 309         pt->count = 0;
 310         pt->pd = pd;
 311         pt->index = 0;
 312 
 313         return pt;
 314 }
 315 
 316 struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
 317                                              unsigned long addr)
 318 {
 319         uint32_t index = psb_mmu_pd_index(addr);
 320         struct psb_mmu_pt *pt;
 321         uint32_t *v;
 322         spinlock_t *lock = &pd->driver->lock;
 323 
 324         spin_lock(lock);
 325         pt = pd->tables[index];
 326         while (!pt) {
 327                 spin_unlock(lock);
 328                 pt = psb_mmu_alloc_pt(pd);
 329                 if (!pt)
 330                         return NULL;
 331                 spin_lock(lock);
 332 
 333                 if (pd->tables[index]) {
 334                         spin_unlock(lock);
 335                         psb_mmu_free_pt(pt);
 336                         spin_lock(lock);
 337                         pt = pd->tables[index];
 338                         continue;
 339                 }
 340 
 341                 v = kmap_atomic(pd->p);
 342                 pd->tables[index] = pt;
 343                 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
 344                 pt->index = index;
 345                 kunmap_atomic((void *) v);
 346 
 347                 if (pd->hw_context != -1) {
 348                         psb_mmu_clflush(pd->driver, (void *)&v[index]);
 349                         atomic_set(&pd->driver->needs_tlbflush, 1);
 350                 }
 351         }
 352         pt->v = kmap_atomic(pt->p);
 353         return pt;
 354 }
 355 
 356 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
 357                                               unsigned long addr)
 358 {
 359         uint32_t index = psb_mmu_pd_index(addr);
 360         struct psb_mmu_pt *pt;
 361         spinlock_t *lock = &pd->driver->lock;
 362 
 363         spin_lock(lock);
 364         pt = pd->tables[index];
 365         if (!pt) {
 366                 spin_unlock(lock);
 367                 return NULL;
 368         }
 369         pt->v = kmap_atomic(pt->p);
 370         return pt;
 371 }
 372 
 373 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
 374 {
 375         struct psb_mmu_pd *pd = pt->pd;
 376         uint32_t *v;
 377 
 378         kunmap_atomic(pt->v);
 379         if (pt->count == 0) {
 380                 v = kmap_atomic(pd->p);
 381                 v[pt->index] = pd->invalid_pde;
 382                 pd->tables[pt->index] = NULL;
 383 
 384                 if (pd->hw_context != -1) {
 385                         psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
 386                         atomic_set(&pd->driver->needs_tlbflush, 1);
 387                 }
 388                 kunmap_atomic(v);
 389                 spin_unlock(&pd->driver->lock);
 390                 psb_mmu_free_pt(pt);
 391                 return;
 392         }
 393         spin_unlock(&pd->driver->lock);
 394 }
 395 
 396 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
 397                                    uint32_t pte)
 398 {
 399         pt->v[psb_mmu_pt_index(addr)] = pte;
 400 }
 401 
 402 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
 403                                           unsigned long addr)
 404 {
 405         pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
 406 }
 407 
 408 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
 409 {
 410         struct psb_mmu_pd *pd;
 411 
 412         down_read(&driver->sem);
 413         pd = driver->default_pd;
 414         up_read(&driver->sem);
 415 
 416         return pd;
 417 }
 418 
 419 /* Returns the physical address of the PD shared by sgx/msvdx */
 420 uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
 421 {
 422         struct psb_mmu_pd *pd;
 423 
 424         pd = psb_mmu_get_default_pd(driver);
 425         return page_to_pfn(pd->p) << PAGE_SHIFT;
 426 }
 427 
 428 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
 429 {
 430         struct drm_device *dev = driver->dev;
 431         struct drm_psb_private *dev_priv = dev->dev_private;
 432 
 433         PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
 434         psb_mmu_free_pagedir(driver->default_pd);
 435         kfree(driver);
 436 }
 437 
 438 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
 439                                            int trap_pagefaults,
 440                                            int invalid_type,
 441                                            atomic_t *msvdx_mmu_invaldc)
 442 {
 443         struct psb_mmu_driver *driver;
 444         struct drm_psb_private *dev_priv = dev->dev_private;
 445 
 446         driver = kmalloc(sizeof(*driver), GFP_KERNEL);
 447 
 448         if (!driver)
 449                 return NULL;
 450 
 451         driver->dev = dev;
 452         driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
 453                                               invalid_type);
 454         if (!driver->default_pd)
 455                 goto out_err1;
 456 
 457         spin_lock_init(&driver->lock);
 458         init_rwsem(&driver->sem);
 459         down_write(&driver->sem);
 460         atomic_set(&driver->needs_tlbflush, 1);
 461         driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
 462 
 463         driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
 464         PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
 465                    PSB_CR_BIF_CTRL);
 466         PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
 467                    PSB_CR_BIF_CTRL);
 468 
 469         driver->has_clflush = 0;
 470 
 471 #if defined(CONFIG_X86)
 472         if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
 473                 uint32_t tfms, misc, cap0, cap4, clflush_size;
 474 
 475                 /*
 476                  * clflush size is determined at kernel setup for x86_64 but not
 477                  * for i386. We have to do it here.
 478                  */
 479 
 480                 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
 481                 clflush_size = ((misc >> 8) & 0xff) * 8;
 482                 driver->has_clflush = 1;
 483                 driver->clflush_add =
 484                     PAGE_SIZE * clflush_size / sizeof(uint32_t);
 485                 driver->clflush_mask = driver->clflush_add - 1;
 486                 driver->clflush_mask = ~driver->clflush_mask;
 487         }
 488 #endif
 489 
 490         up_write(&driver->sem);
 491         return driver;
 492 
 493 out_err1:
 494         kfree(driver);
 495         return NULL;
 496 }
 497 
 498 #if defined(CONFIG_X86)
 499 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 500                                uint32_t num_pages, uint32_t desired_tile_stride,
 501                                uint32_t hw_tile_stride)
 502 {
 503         struct psb_mmu_pt *pt;
 504         uint32_t rows = 1;
 505         uint32_t i;
 506         unsigned long addr;
 507         unsigned long end;
 508         unsigned long next;
 509         unsigned long add;
 510         unsigned long row_add;
 511         unsigned long clflush_add = pd->driver->clflush_add;
 512         unsigned long clflush_mask = pd->driver->clflush_mask;
 513 
 514         if (!pd->driver->has_clflush)
 515                 return;
 516 
 517         if (hw_tile_stride)
 518                 rows = num_pages / desired_tile_stride;
 519         else
 520                 desired_tile_stride = num_pages;
 521 
 522         add = desired_tile_stride << PAGE_SHIFT;
 523         row_add = hw_tile_stride << PAGE_SHIFT;
 524         mb();
 525         for (i = 0; i < rows; ++i) {
 526 
 527                 addr = address;
 528                 end = addr + add;
 529 
 530                 do {
 531                         next = psb_pd_addr_end(addr, end);
 532                         pt = psb_mmu_pt_map_lock(pd, addr);
 533                         if (!pt)
 534                                 continue;
 535                         do {
 536                                 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
 537                         } while (addr += clflush_add,
 538                                  (addr & clflush_mask) < next);
 539 
 540                         psb_mmu_pt_unmap_unlock(pt);
 541                 } while (addr = next, next != end);
 542                 address += row_add;
 543         }
 544         mb();
 545 }
 546 #else
 547 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 548                                uint32_t num_pages, uint32_t desired_tile_stride,
 549                                uint32_t hw_tile_stride)
 550 {
 551         drm_ttm_cache_flush();
 552 }
 553 #endif
 554 
 555 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
 556                                  unsigned long address, uint32_t num_pages)
 557 {
 558         struct psb_mmu_pt *pt;
 559         unsigned long addr;
 560         unsigned long end;
 561         unsigned long next;
 562         unsigned long f_address = address;
 563 
 564         down_read(&pd->driver->sem);
 565 
 566         addr = address;
 567         end = addr + (num_pages << PAGE_SHIFT);
 568 
 569         do {
 570                 next = psb_pd_addr_end(addr, end);
 571                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 572                 if (!pt)
 573                         goto out;
 574                 do {
 575                         psb_mmu_invalidate_pte(pt, addr);
 576                         --pt->count;
 577                 } while (addr += PAGE_SIZE, addr < next);
 578                 psb_mmu_pt_unmap_unlock(pt);
 579 
 580         } while (addr = next, next != end);
 581 
 582 out:
 583         if (pd->hw_context != -1)
 584                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 585 
 586         up_read(&pd->driver->sem);
 587 
 588         if (pd->hw_context != -1)
 589                 psb_mmu_flush(pd->driver);
 590 
 591         return;
 592 }
 593 
 594 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
 595                           uint32_t num_pages, uint32_t desired_tile_stride,
 596                           uint32_t hw_tile_stride)
 597 {
 598         struct psb_mmu_pt *pt;
 599         uint32_t rows = 1;
 600         uint32_t i;
 601         unsigned long addr;
 602         unsigned long end;
 603         unsigned long next;
 604         unsigned long add;
 605         unsigned long row_add;
 606         unsigned long f_address = address;
 607 
 608         if (hw_tile_stride)
 609                 rows = num_pages / desired_tile_stride;
 610         else
 611                 desired_tile_stride = num_pages;
 612 
 613         add = desired_tile_stride << PAGE_SHIFT;
 614         row_add = hw_tile_stride << PAGE_SHIFT;
 615 
 616         down_read(&pd->driver->sem);
 617 
 618         /* Make sure we only need to flush this processor's cache */
 619 
 620         for (i = 0; i < rows; ++i) {
 621 
 622                 addr = address;
 623                 end = addr + add;
 624 
 625                 do {
 626                         next = psb_pd_addr_end(addr, end);
 627                         pt = psb_mmu_pt_map_lock(pd, addr);
 628                         if (!pt)
 629                                 continue;
 630                         do {
 631                                 psb_mmu_invalidate_pte(pt, addr);
 632                                 --pt->count;
 633 
 634                         } while (addr += PAGE_SIZE, addr < next);
 635                         psb_mmu_pt_unmap_unlock(pt);
 636 
 637                 } while (addr = next, next != end);
 638                 address += row_add;
 639         }
 640         if (pd->hw_context != -1)
 641                 psb_mmu_flush_ptes(pd, f_address, num_pages,
 642                                    desired_tile_stride, hw_tile_stride);
 643 
 644         up_read(&pd->driver->sem);
 645 
 646         if (pd->hw_context != -1)
 647                 psb_mmu_flush(pd->driver);
 648 }
 649 
 650 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
 651                                 unsigned long address, uint32_t num_pages,
 652                                 int type)
 653 {
 654         struct psb_mmu_pt *pt;
 655         uint32_t pte;
 656         unsigned long addr;
 657         unsigned long end;
 658         unsigned long next;
 659         unsigned long f_address = address;
 660         int ret = -ENOMEM;
 661 
 662         down_read(&pd->driver->sem);
 663 
 664         addr = address;
 665         end = addr + (num_pages << PAGE_SHIFT);
 666 
 667         do {
 668                 next = psb_pd_addr_end(addr, end);
 669                 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 670                 if (!pt) {
 671                         ret = -ENOMEM;
 672                         goto out;
 673                 }
 674                 do {
 675                         pte = psb_mmu_mask_pte(start_pfn++, type);
 676                         psb_mmu_set_pte(pt, addr, pte);
 677                         pt->count++;
 678                 } while (addr += PAGE_SIZE, addr < next);
 679                 psb_mmu_pt_unmap_unlock(pt);
 680 
 681         } while (addr = next, next != end);
 682         ret = 0;
 683 
 684 out:
 685         if (pd->hw_context != -1)
 686                 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 687 
 688         up_read(&pd->driver->sem);
 689 
 690         if (pd->hw_context != -1)
 691                 psb_mmu_flush(pd->driver);
 692 
 693         return 0;
 694 }
 695 
 696 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
 697                          unsigned long address, uint32_t num_pages,
 698                          uint32_t desired_tile_stride, uint32_t hw_tile_stride,
 699                          int type)
 700 {
 701         struct psb_mmu_pt *pt;
 702         uint32_t rows = 1;
 703         uint32_t i;
 704         uint32_t pte;
 705         unsigned long addr;
 706         unsigned long end;
 707         unsigned long next;
 708         unsigned long add;
 709         unsigned long row_add;
 710         unsigned long f_address = address;
 711         int ret = -ENOMEM;
 712 
 713         if (hw_tile_stride) {
 714                 if (num_pages % desired_tile_stride != 0)
 715                         return -EINVAL;
 716                 rows = num_pages / desired_tile_stride;
 717         } else {
 718                 desired_tile_stride = num_pages;
 719         }
 720 
 721         add = desired_tile_stride << PAGE_SHIFT;
 722         row_add = hw_tile_stride << PAGE_SHIFT;
 723 
 724         down_read(&pd->driver->sem);
 725 
 726         for (i = 0; i < rows; ++i) {
 727 
 728                 addr = address;
 729                 end = addr + add;
 730 
 731                 do {
 732                         next = psb_pd_addr_end(addr, end);
 733                         pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 734                         if (!pt)
 735                                 goto out;
 736                         do {
 737                                 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
 738                                                        type);
 739                                 psb_mmu_set_pte(pt, addr, pte);
 740                                 pt->count++;
 741                         } while (addr += PAGE_SIZE, addr < next);
 742                         psb_mmu_pt_unmap_unlock(pt);
 743 
 744                 } while (addr = next, next != end);
 745 
 746                 address += row_add;
 747         }
 748 
 749         ret = 0;
 750 out:
 751         if (pd->hw_context != -1)
 752                 psb_mmu_flush_ptes(pd, f_address, num_pages,
 753                                    desired_tile_stride, hw_tile_stride);
 754 
 755         up_read(&pd->driver->sem);
 756 
 757         if (pd->hw_context != -1)
 758                 psb_mmu_flush(pd->driver);
 759 
 760         return ret;
 761 }
 762 
 763 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
 764                            unsigned long *pfn)
 765 {
 766         int ret;
 767         struct psb_mmu_pt *pt;
 768         uint32_t tmp;
 769         spinlock_t *lock = &pd->driver->lock;
 770 
 771         down_read(&pd->driver->sem);
 772         pt = psb_mmu_pt_map_lock(pd, virtual);
 773         if (!pt) {
 774                 uint32_t *v;
 775 
 776                 spin_lock(lock);
 777                 v = kmap_atomic(pd->p);
 778                 tmp = v[psb_mmu_pd_index(virtual)];
 779                 kunmap_atomic(v);
 780                 spin_unlock(lock);
 781 
 782                 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
 783                     !(pd->invalid_pte & PSB_PTE_VALID)) {
 784                         ret = -EINVAL;
 785                         goto out;
 786                 }
 787                 ret = 0;
 788                 *pfn = pd->invalid_pte >> PAGE_SHIFT;
 789                 goto out;
 790         }
 791         tmp = pt->v[psb_mmu_pt_index(virtual)];
 792         if (!(tmp & PSB_PTE_VALID)) {
 793                 ret = -EINVAL;
 794         } else {
 795                 ret = 0;
 796                 *pfn = tmp >> PAGE_SHIFT;
 797         }
 798         psb_mmu_pt_unmap_unlock(pt);
 799 out:
 800         up_read(&pd->driver->sem);
 801         return ret;
 802 }