1/* arch/sparc64/mm/tsb.c 2 * 3 * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net> 4 */ 5 6#include <linux/kernel.h> 7#include <linux/preempt.h> 8#include <linux/slab.h> 9#include <asm/page.h> 10#include <asm/pgtable.h> 11#include <asm/mmu_context.h> 12#include <asm/setup.h> 13#include <asm/tsb.h> 14#include <asm/tlb.h> 15#include <asm/oplib.h> 16 17extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; 18 19static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries) 20{ 21 vaddr >>= hash_shift; 22 return vaddr & (nentries - 1); 23} 24 25static inline int tag_compare(unsigned long tag, unsigned long vaddr) 26{ 27 return (tag == (vaddr >> 22)); 28} 29 30/* TSB flushes need only occur on the processor initiating the address 31 * space modification, not on each cpu the address space has run on. 32 * Only the TLB flush needs that treatment. 33 */ 34 35void flush_tsb_kernel_range(unsigned long start, unsigned long end) 36{ 37 unsigned long v; 38 39 for (v = start; v < end; v += PAGE_SIZE) { 40 unsigned long hash = tsb_hash(v, PAGE_SHIFT, 41 KERNEL_TSB_NENTRIES); 42 struct tsb *ent = &swapper_tsb[hash]; 43 44 if (tag_compare(ent->tag, v)) 45 ent->tag = (1UL << TSB_TAG_INVALID_BIT); 46 } 47} 48 49static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v, 50 unsigned long hash_shift, 51 unsigned long nentries) 52{ 53 unsigned long tag, ent, hash; 54 55 v &= ~0x1UL; 56 hash = tsb_hash(v, hash_shift, nentries); 57 ent = tsb + (hash * sizeof(struct tsb)); 58 tag = (v >> 22UL); 59 60 tsb_flush(ent, tag); 61} 62 63static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, 64 unsigned long tsb, unsigned long nentries) 65{ 66 unsigned long i; 67 68 for (i = 0; i < tb->tlb_nr; i++) 69 __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); 70} 71 72void flush_tsb_user(struct tlb_batch *tb) 73{ 74 struct mm_struct *mm = tb->mm; 75 unsigned long nentries, base, flags; 76 77 spin_lock_irqsave(&mm->context.lock, flags); 78 79 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; 80 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; 81 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 82 base = __pa(base); 83 __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); 84 85#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 86 if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { 87 base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; 88 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; 89 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 90 base = __pa(base); 91 __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries); 92 } 93#endif 94 spin_unlock_irqrestore(&mm->context.lock, flags); 95} 96 97void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr) 98{ 99 unsigned long nentries, base, flags; 100 101 spin_lock_irqsave(&mm->context.lock, flags); 102 103 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; 104 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; 105 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 106 base = __pa(base); 107 __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); 108 109#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 110 if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { 111 base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; 112 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; 113 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 114 base = __pa(base); 115 __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries); 116 } 117#endif 118 spin_unlock_irqrestore(&mm->context.lock, flags); 119} 120 121#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K 122#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K 123 124#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 125#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB 126#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB 127#endif 128 129static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes) 130{ 131 unsigned long tsb_reg, base, tsb_paddr; 132 unsigned long page_sz, tte; 133 134 mm->context.tsb_block[tsb_idx].tsb_nentries = 135 tsb_bytes / sizeof(struct tsb); 136 137 switch (tsb_idx) { 138 case MM_TSB_BASE: 139 base = TSBMAP_8K_BASE; 140 break; 141#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 142 case MM_TSB_HUGE: 143 base = TSBMAP_4M_BASE; 144 break; 145#endif 146 default: 147 BUG(); 148 } 149 150 tte = pgprot_val(PAGE_KERNEL_LOCKED); 151 tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb); 152 BUG_ON(tsb_paddr & (tsb_bytes - 1UL)); 153 154 /* Use the smallest page size that can map the whole TSB 155 * in one TLB entry. 156 */ 157 switch (tsb_bytes) { 158 case 8192 << 0: 159 tsb_reg = 0x0UL; 160#ifdef DCACHE_ALIASING_POSSIBLE 161 base += (tsb_paddr & 8192); 162#endif 163 page_sz = 8192; 164 break; 165 166 case 8192 << 1: 167 tsb_reg = 0x1UL; 168 page_sz = 64 * 1024; 169 break; 170 171 case 8192 << 2: 172 tsb_reg = 0x2UL; 173 page_sz = 64 * 1024; 174 break; 175 176 case 8192 << 3: 177 tsb_reg = 0x3UL; 178 page_sz = 64 * 1024; 179 break; 180 181 case 8192 << 4: 182 tsb_reg = 0x4UL; 183 page_sz = 512 * 1024; 184 break; 185 186 case 8192 << 5: 187 tsb_reg = 0x5UL; 188 page_sz = 512 * 1024; 189 break; 190 191 case 8192 << 6: 192 tsb_reg = 0x6UL; 193 page_sz = 512 * 1024; 194 break; 195 196 case 8192 << 7: 197 tsb_reg = 0x7UL; 198 page_sz = 4 * 1024 * 1024; 199 break; 200 201 default: 202 printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n", 203 current->comm, current->pid, tsb_bytes); 204 do_exit(SIGSEGV); 205 } 206 tte |= pte_sz_bits(page_sz); 207 208 if (tlb_type == cheetah_plus || tlb_type == hypervisor) { 209 /* Physical mapping, no locked TLB entry for TSB. */ 210 tsb_reg |= tsb_paddr; 211 212 mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg; 213 mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0; 214 mm->context.tsb_block[tsb_idx].tsb_map_pte = 0; 215 } else { 216 tsb_reg |= base; 217 tsb_reg |= (tsb_paddr & (page_sz - 1UL)); 218 tte |= (tsb_paddr & ~(page_sz - 1UL)); 219 220 mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg; 221 mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base; 222 mm->context.tsb_block[tsb_idx].tsb_map_pte = tte; 223 } 224 225 /* Setup the Hypervisor TSB descriptor. */ 226 if (tlb_type == hypervisor) { 227 struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx]; 228 229 switch (tsb_idx) { 230 case MM_TSB_BASE: 231 hp->pgsz_idx = HV_PGSZ_IDX_BASE; 232 break; 233#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 234 case MM_TSB_HUGE: 235 hp->pgsz_idx = HV_PGSZ_IDX_HUGE; 236 break; 237#endif 238 default: 239 BUG(); 240 } 241 hp->assoc = 1; 242 hp->num_ttes = tsb_bytes / 16; 243 hp->ctx_idx = 0; 244 switch (tsb_idx) { 245 case MM_TSB_BASE: 246 hp->pgsz_mask = HV_PGSZ_MASK_BASE; 247 break; 248#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 249 case MM_TSB_HUGE: 250 hp->pgsz_mask = HV_PGSZ_MASK_HUGE; 251 break; 252#endif 253 default: 254 BUG(); 255 } 256 hp->tsb_base = tsb_paddr; 257 hp->resv = 0; 258 } 259} 260 261struct kmem_cache *pgtable_cache __read_mostly; 262 263static struct kmem_cache *tsb_caches[8] __read_mostly; 264 265static const char *tsb_cache_names[8] = { 266 "tsb_8KB", 267 "tsb_16KB", 268 "tsb_32KB", 269 "tsb_64KB", 270 "tsb_128KB", 271 "tsb_256KB", 272 "tsb_512KB", 273 "tsb_1MB", 274}; 275 276void __init pgtable_cache_init(void) 277{ 278 unsigned long i; 279 280 pgtable_cache = kmem_cache_create("pgtable_cache", 281 PAGE_SIZE, PAGE_SIZE, 282 0, 283 _clear_page); 284 if (!pgtable_cache) { 285 prom_printf("pgtable_cache_init(): Could not create!\n"); 286 prom_halt(); 287 } 288 289 for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) { 290 unsigned long size = 8192 << i; 291 const char *name = tsb_cache_names[i]; 292 293 tsb_caches[i] = kmem_cache_create(name, 294 size, size, 295 0, NULL); 296 if (!tsb_caches[i]) { 297 prom_printf("Could not create %s cache\n", name); 298 prom_halt(); 299 } 300 } 301} 302 303int sysctl_tsb_ratio = -2; 304 305static unsigned long tsb_size_to_rss_limit(unsigned long new_size) 306{ 307 unsigned long num_ents = (new_size / sizeof(struct tsb)); 308 309 if (sysctl_tsb_ratio < 0) 310 return num_ents - (num_ents >> -sysctl_tsb_ratio); 311 else 312 return num_ents + (num_ents >> sysctl_tsb_ratio); 313} 314 315/* When the RSS of an address space exceeds tsb_rss_limit for a TSB, 316 * do_sparc64_fault() invokes this routine to try and grow it. 317 * 318 * When we reach the maximum TSB size supported, we stick ~0UL into 319 * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault() 320 * will not trigger any longer. 321 * 322 * The TSB can be anywhere from 8K to 1MB in size, in increasing powers 323 * of two. The TSB must be aligned to it's size, so f.e. a 512K TSB 324 * must be 512K aligned. It also must be physically contiguous, so we 325 * cannot use vmalloc(). 326 * 327 * The idea here is to grow the TSB when the RSS of the process approaches 328 * the number of entries that the current TSB can hold at once. Currently, 329 * we trigger when the RSS hits 3/4 of the TSB capacity. 330 */ 331void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) 332{ 333 unsigned long max_tsb_size = 1 * 1024 * 1024; 334 unsigned long new_size, old_size, flags; 335 struct tsb *old_tsb, *new_tsb; 336 unsigned long new_cache_index, old_cache_index; 337 unsigned long new_rss_limit; 338 gfp_t gfp_flags; 339 340 if (max_tsb_size > (PAGE_SIZE << MAX_ORDER)) 341 max_tsb_size = (PAGE_SIZE << MAX_ORDER); 342 343 new_cache_index = 0; 344 for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) { 345 new_rss_limit = tsb_size_to_rss_limit(new_size); 346 if (new_rss_limit > rss) 347 break; 348 new_cache_index++; 349 } 350 351 if (new_size == max_tsb_size) 352 new_rss_limit = ~0UL; 353 354retry_tsb_alloc: 355 gfp_flags = GFP_KERNEL; 356 if (new_size > (PAGE_SIZE * 2)) 357 gfp_flags |= __GFP_NOWARN | __GFP_NORETRY; 358 359 new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index], 360 gfp_flags, numa_node_id()); 361 if (unlikely(!new_tsb)) { 362 /* Not being able to fork due to a high-order TSB 363 * allocation failure is very bad behavior. Just back 364 * down to a 0-order allocation and force no TSB 365 * growing for this address space. 366 */ 367 if (mm->context.tsb_block[tsb_index].tsb == NULL && 368 new_cache_index > 0) { 369 new_cache_index = 0; 370 new_size = 8192; 371 new_rss_limit = ~0UL; 372 goto retry_tsb_alloc; 373 } 374 375 /* If we failed on a TSB grow, we are under serious 376 * memory pressure so don't try to grow any more. 377 */ 378 if (mm->context.tsb_block[tsb_index].tsb != NULL) 379 mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL; 380 return; 381 } 382 383 /* Mark all tags as invalid. */ 384 tsb_init(new_tsb, new_size); 385 386 /* Ok, we are about to commit the changes. If we are 387 * growing an existing TSB the locking is very tricky, 388 * so WATCH OUT! 389 * 390 * We have to hold mm->context.lock while committing to the 391 * new TSB, this synchronizes us with processors in 392 * flush_tsb_user() and switch_mm() for this address space. 393 * 394 * But even with that lock held, processors run asynchronously 395 * accessing the old TSB via TLB miss handling. This is OK 396 * because those actions are just propagating state from the 397 * Linux page tables into the TSB, page table mappings are not 398 * being changed. If a real fault occurs, the processor will 399 * synchronize with us when it hits flush_tsb_user(), this is 400 * also true for the case where vmscan is modifying the page 401 * tables. The only thing we need to be careful with is to 402 * skip any locked TSB entries during copy_tsb(). 403 * 404 * When we finish committing to the new TSB, we have to drop 405 * the lock and ask all other cpus running this address space 406 * to run tsb_context_switch() to see the new TSB table. 407 */ 408 spin_lock_irqsave(&mm->context.lock, flags); 409 410 old_tsb = mm->context.tsb_block[tsb_index].tsb; 411 old_cache_index = 412 (mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL); 413 old_size = (mm->context.tsb_block[tsb_index].tsb_nentries * 414 sizeof(struct tsb)); 415 416 417 /* Handle multiple threads trying to grow the TSB at the same time. 418 * One will get in here first, and bump the size and the RSS limit. 419 * The others will get in here next and hit this check. 420 */ 421 if (unlikely(old_tsb && 422 (rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) { 423 spin_unlock_irqrestore(&mm->context.lock, flags); 424 425 kmem_cache_free(tsb_caches[new_cache_index], new_tsb); 426 return; 427 } 428 429 mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit; 430 431 if (old_tsb) { 432 extern void copy_tsb(unsigned long old_tsb_base, 433 unsigned long old_tsb_size, 434 unsigned long new_tsb_base, 435 unsigned long new_tsb_size); 436 unsigned long old_tsb_base = (unsigned long) old_tsb; 437 unsigned long new_tsb_base = (unsigned long) new_tsb; 438 439 if (tlb_type == cheetah_plus || tlb_type == hypervisor) { 440 old_tsb_base = __pa(old_tsb_base); 441 new_tsb_base = __pa(new_tsb_base); 442 } 443 copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size); 444 } 445 446 mm->context.tsb_block[tsb_index].tsb = new_tsb; 447 setup_tsb_params(mm, tsb_index, new_size); 448 449 spin_unlock_irqrestore(&mm->context.lock, flags); 450 451 /* If old_tsb is NULL, we're being invoked for the first time 452 * from init_new_context(). 453 */ 454 if (old_tsb) { 455 /* Reload it on the local cpu. */ 456 tsb_context_switch(mm); 457 458 /* Now force other processors to do the same. */ 459 preempt_disable(); 460 smp_tsb_sync(mm); 461 preempt_enable(); 462 463 /* Now it is safe to free the old tsb. */ 464 kmem_cache_free(tsb_caches[old_cache_index], old_tsb); 465 } 466} 467 468int init_new_context(struct task_struct *tsk, struct mm_struct *mm) 469{ 470#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 471 unsigned long huge_pte_count; 472#endif 473 unsigned int i; 474 475 spin_lock_init(&mm->context.lock); 476 477 mm->context.sparc64_ctx_val = 0UL; 478 479#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 480 /* We reset it to zero because the fork() page copying 481 * will re-increment the counters as the parent PTEs are 482 * copied into the child address space. 483 */ 484 huge_pte_count = mm->context.huge_pte_count; 485 mm->context.huge_pte_count = 0; 486#endif 487 488 /* copy_mm() copies over the parent's mm_struct before calling 489 * us, so we need to zero out the TSB pointer or else tsb_grow() 490 * will be confused and think there is an older TSB to free up. 491 */ 492 for (i = 0; i < MM_NUM_TSBS; i++) 493 mm->context.tsb_block[i].tsb = NULL; 494 495 /* If this is fork, inherit the parent's TSB size. We would 496 * grow it to that size on the first page fault anyways. 497 */ 498 tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); 499 500#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 501 if (unlikely(huge_pte_count)) 502 tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); 503#endif 504 505 if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) 506 return -ENOMEM; 507 508 return 0; 509} 510 511static void tsb_destroy_one(struct tsb_config *tp) 512{ 513 unsigned long cache_index; 514 515 if (!tp->tsb) 516 return; 517 cache_index = tp->tsb_reg_val & 0x7UL; 518 kmem_cache_free(tsb_caches[cache_index], tp->tsb); 519 tp->tsb = NULL; 520 tp->tsb_reg_val = 0UL; 521} 522 523void destroy_context(struct mm_struct *mm) 524{ 525 unsigned long flags, i; 526 527 for (i = 0; i < MM_NUM_TSBS; i++) 528 tsb_destroy_one(&mm->context.tsb_block[i]); 529 530 spin_lock_irqsave(&ctx_alloc_lock, flags); 531 532 if (CTX_VALID(mm->context)) { 533 unsigned long nr = CTX_NRBITS(mm->context); 534 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63)); 535 } 536 537 spin_unlock_irqrestore(&ctx_alloc_lock, flags); 538} 539