1/* 2 * Common EFI (Extensible Firmware Interface) support functions 3 * Based on Extensible Firmware Interface Specification version 1.0 4 * 5 * Copyright (C) 1999 VA Linux Systems 6 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 7 * Copyright (C) 1999-2002 Hewlett-Packard Co. 8 * David Mosberger-Tang <davidm@hpl.hp.com> 9 * Stephane Eranian <eranian@hpl.hp.com> 10 * Copyright (C) 2005-2008 Intel Co. 11 * Fenghua Yu <fenghua.yu@intel.com> 12 * Bibo Mao <bibo.mao@intel.com> 13 * Chandramouli Narayanan <mouli@linux.intel.com> 14 * Huang Ying <ying.huang@intel.com> 15 * Copyright (C) 2013 SuSE Labs 16 * Borislav Petkov <bp@suse.de> - runtime services VA mapping 17 * 18 * Copied from efi_32.c to eliminate the duplicated code between EFI 19 * 32/64 support code. --ying 2007-10-26 20 * 21 * All EFI Runtime Services are not implemented yet as EFI only 22 * supports physical mode addressing on SoftSDV. This is to be fixed 23 * in a future version. --drummond 1999-07-20 24 * 25 * Implemented EFI runtime services and virtual mode calls. --davidm 26 * 27 * Goutham Rao: <goutham.rao@intel.com> 28 * Skip non-WB memory and ignore empty memory ranges. 29 */ 30 31#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33#include <linux/kernel.h> 34#include <linux/init.h> 35#include <linux/efi.h> 36#include <linux/efi-bgrt.h> 37#include <linux/export.h> 38#include <linux/bootmem.h> 39#include <linux/slab.h> 40#include <linux/memblock.h> 41#include <linux/spinlock.h> 42#include <linux/uaccess.h> 43#include <linux/time.h> 44#include <linux/io.h> 45#include <linux/reboot.h> 46#include <linux/bcd.h> 47 48#include <asm/setup.h> 49#include <asm/efi.h> 50#include <asm/time.h> 51#include <asm/cacheflush.h> 52#include <asm/tlbflush.h> 53#include <asm/x86_init.h> 54#include <asm/rtc.h> 55#include <asm/uv/uv.h> 56 57#define EFI_DEBUG 58 59struct efi_memory_map memmap; 60 61static struct efi efi_phys __initdata; 62static efi_system_table_t efi_systab __initdata; 63 64static efi_config_table_type_t arch_tables[] __initdata = { 65#ifdef CONFIG_X86_UV 66 {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab}, 67#endif 68 {NULL_GUID, NULL, NULL}, 69}; 70 71u64 efi_setup; /* efi setup_data physical address */ 72 73static int add_efi_memmap __initdata; 74static int __init setup_add_efi_memmap(char *arg) 75{ 76 add_efi_memmap = 1; 77 return 0; 78} 79early_param("add_efi_memmap", setup_add_efi_memmap); 80 81static efi_status_t __init phys_efi_set_virtual_address_map( 82 unsigned long memory_map_size, 83 unsigned long descriptor_size, 84 u32 descriptor_version, 85 efi_memory_desc_t *virtual_map) 86{ 87 efi_status_t status; 88 unsigned long flags; 89 pgd_t *save_pgd; 90 91 save_pgd = efi_call_phys_prolog(); 92 93 /* Disable interrupts around EFI calls: */ 94 local_irq_save(flags); 95 status = efi_call_phys(efi_phys.set_virtual_address_map, 96 memory_map_size, descriptor_size, 97 descriptor_version, virtual_map); 98 local_irq_restore(flags); 99 100 efi_call_phys_epilog(save_pgd); 101 102 return status; 103} 104 105void efi_get_time(struct timespec *now) 106{ 107 efi_status_t status; 108 efi_time_t eft; 109 efi_time_cap_t cap; 110 111 status = efi.get_time(&eft, &cap); 112 if (status != EFI_SUCCESS) 113 pr_err("Oops: efitime: can't read time!\n"); 114 115 now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour, 116 eft.minute, eft.second); 117 now->tv_nsec = 0; 118} 119 120/* 121 * Tell the kernel about the EFI memory map. This might include 122 * more than the max 128 entries that can fit in the e820 legacy 123 * (zeropage) memory map. 124 */ 125 126static void __init do_add_efi_memmap(void) 127{ 128 void *p; 129 130 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 131 efi_memory_desc_t *md = p; 132 unsigned long long start = md->phys_addr; 133 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 134 int e820_type; 135 136 switch (md->type) { 137 case EFI_LOADER_CODE: 138 case EFI_LOADER_DATA: 139 case EFI_BOOT_SERVICES_CODE: 140 case EFI_BOOT_SERVICES_DATA: 141 case EFI_CONVENTIONAL_MEMORY: 142 if (md->attribute & EFI_MEMORY_WB) 143 e820_type = E820_RAM; 144 else 145 e820_type = E820_RESERVED; 146 break; 147 case EFI_ACPI_RECLAIM_MEMORY: 148 e820_type = E820_ACPI; 149 break; 150 case EFI_ACPI_MEMORY_NVS: 151 e820_type = E820_NVS; 152 break; 153 case EFI_UNUSABLE_MEMORY: 154 e820_type = E820_UNUSABLE; 155 break; 156 default: 157 /* 158 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE 159 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO 160 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE 161 */ 162 e820_type = E820_RESERVED; 163 break; 164 } 165 e820_add_region(start, size, e820_type); 166 } 167 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map); 168} 169 170int __init efi_memblock_x86_reserve_range(void) 171{ 172 struct efi_info *e = &boot_params.efi_info; 173 unsigned long pmap; 174 175 if (efi_enabled(EFI_PARAVIRT)) 176 return 0; 177 178#ifdef CONFIG_X86_32 179 /* Can't handle data above 4GB at this time */ 180 if (e->efi_memmap_hi) { 181 pr_err("Memory map is above 4GB, disabling EFI.\n"); 182 return -EINVAL; 183 } 184 pmap = e->efi_memmap; 185#else 186 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); 187#endif 188 memmap.phys_map = (void *)pmap; 189 memmap.nr_map = e->efi_memmap_size / 190 e->efi_memdesc_size; 191 memmap.desc_size = e->efi_memdesc_size; 192 memmap.desc_version = e->efi_memdesc_version; 193 194 memblock_reserve(pmap, memmap.nr_map * memmap.desc_size); 195 196 efi.memmap = &memmap; 197 198 return 0; 199} 200 201static void __init print_efi_memmap(void) 202{ 203#ifdef EFI_DEBUG 204 efi_memory_desc_t *md; 205 void *p; 206 int i; 207 208 for (p = memmap.map, i = 0; 209 p < memmap.map_end; 210 p += memmap.desc_size, i++) { 211 char buf[64]; 212 213 md = p; 214 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx) (%lluMB)\n", 215 i, efi_md_typeattr_format(buf, sizeof(buf), md), 216 md->phys_addr, 217 md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT), 218 (md->num_pages >> (20 - EFI_PAGE_SHIFT))); 219 } 220#endif /* EFI_DEBUG */ 221} 222 223void __init efi_unmap_memmap(void) 224{ 225 clear_bit(EFI_MEMMAP, &efi.flags); 226 if (memmap.map) { 227 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size); 228 memmap.map = NULL; 229 } 230} 231 232static int __init efi_systab_init(void *phys) 233{ 234 if (efi_enabled(EFI_64BIT)) { 235 efi_system_table_64_t *systab64; 236 struct efi_setup_data *data = NULL; 237 u64 tmp = 0; 238 239 if (efi_setup) { 240 data = early_memremap(efi_setup, sizeof(*data)); 241 if (!data) 242 return -ENOMEM; 243 } 244 systab64 = early_memremap((unsigned long)phys, 245 sizeof(*systab64)); 246 if (systab64 == NULL) { 247 pr_err("Couldn't map the system table!\n"); 248 if (data) 249 early_memunmap(data, sizeof(*data)); 250 return -ENOMEM; 251 } 252 253 efi_systab.hdr = systab64->hdr; 254 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor : 255 systab64->fw_vendor; 256 tmp |= data ? data->fw_vendor : systab64->fw_vendor; 257 efi_systab.fw_revision = systab64->fw_revision; 258 efi_systab.con_in_handle = systab64->con_in_handle; 259 tmp |= systab64->con_in_handle; 260 efi_systab.con_in = systab64->con_in; 261 tmp |= systab64->con_in; 262 efi_systab.con_out_handle = systab64->con_out_handle; 263 tmp |= systab64->con_out_handle; 264 efi_systab.con_out = systab64->con_out; 265 tmp |= systab64->con_out; 266 efi_systab.stderr_handle = systab64->stderr_handle; 267 tmp |= systab64->stderr_handle; 268 efi_systab.stderr = systab64->stderr; 269 tmp |= systab64->stderr; 270 efi_systab.runtime = data ? 271 (void *)(unsigned long)data->runtime : 272 (void *)(unsigned long)systab64->runtime; 273 tmp |= data ? data->runtime : systab64->runtime; 274 efi_systab.boottime = (void *)(unsigned long)systab64->boottime; 275 tmp |= systab64->boottime; 276 efi_systab.nr_tables = systab64->nr_tables; 277 efi_systab.tables = data ? (unsigned long)data->tables : 278 systab64->tables; 279 tmp |= data ? data->tables : systab64->tables; 280 281 early_memunmap(systab64, sizeof(*systab64)); 282 if (data) 283 early_memunmap(data, sizeof(*data)); 284#ifdef CONFIG_X86_32 285 if (tmp >> 32) { 286 pr_err("EFI data located above 4GB, disabling EFI.\n"); 287 return -EINVAL; 288 } 289#endif 290 } else { 291 efi_system_table_32_t *systab32; 292 293 systab32 = early_memremap((unsigned long)phys, 294 sizeof(*systab32)); 295 if (systab32 == NULL) { 296 pr_err("Couldn't map the system table!\n"); 297 return -ENOMEM; 298 } 299 300 efi_systab.hdr = systab32->hdr; 301 efi_systab.fw_vendor = systab32->fw_vendor; 302 efi_systab.fw_revision = systab32->fw_revision; 303 efi_systab.con_in_handle = systab32->con_in_handle; 304 efi_systab.con_in = systab32->con_in; 305 efi_systab.con_out_handle = systab32->con_out_handle; 306 efi_systab.con_out = systab32->con_out; 307 efi_systab.stderr_handle = systab32->stderr_handle; 308 efi_systab.stderr = systab32->stderr; 309 efi_systab.runtime = (void *)(unsigned long)systab32->runtime; 310 efi_systab.boottime = (void *)(unsigned long)systab32->boottime; 311 efi_systab.nr_tables = systab32->nr_tables; 312 efi_systab.tables = systab32->tables; 313 314 early_memunmap(systab32, sizeof(*systab32)); 315 } 316 317 efi.systab = &efi_systab; 318 319 /* 320 * Verify the EFI Table 321 */ 322 if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { 323 pr_err("System table signature incorrect!\n"); 324 return -EINVAL; 325 } 326 if ((efi.systab->hdr.revision >> 16) == 0) 327 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n", 328 efi.systab->hdr.revision >> 16, 329 efi.systab->hdr.revision & 0xffff); 330 331 set_bit(EFI_SYSTEM_TABLES, &efi.flags); 332 333 return 0; 334} 335 336static int __init efi_runtime_init32(void) 337{ 338 efi_runtime_services_32_t *runtime; 339 340 runtime = early_memremap((unsigned long)efi.systab->runtime, 341 sizeof(efi_runtime_services_32_t)); 342 if (!runtime) { 343 pr_err("Could not map the runtime service table!\n"); 344 return -ENOMEM; 345 } 346 347 /* 348 * We will only need *early* access to the SetVirtualAddressMap 349 * EFI runtime service. All other runtime services will be called 350 * via the virtual mapping. 351 */ 352 efi_phys.set_virtual_address_map = 353 (efi_set_virtual_address_map_t *) 354 (unsigned long)runtime->set_virtual_address_map; 355 early_memunmap(runtime, sizeof(efi_runtime_services_32_t)); 356 357 return 0; 358} 359 360static int __init efi_runtime_init64(void) 361{ 362 efi_runtime_services_64_t *runtime; 363 364 runtime = early_memremap((unsigned long)efi.systab->runtime, 365 sizeof(efi_runtime_services_64_t)); 366 if (!runtime) { 367 pr_err("Could not map the runtime service table!\n"); 368 return -ENOMEM; 369 } 370 371 /* 372 * We will only need *early* access to the SetVirtualAddressMap 373 * EFI runtime service. All other runtime services will be called 374 * via the virtual mapping. 375 */ 376 efi_phys.set_virtual_address_map = 377 (efi_set_virtual_address_map_t *) 378 (unsigned long)runtime->set_virtual_address_map; 379 early_memunmap(runtime, sizeof(efi_runtime_services_64_t)); 380 381 return 0; 382} 383 384static int __init efi_runtime_init(void) 385{ 386 int rv; 387 388 /* 389 * Check out the runtime services table. We need to map 390 * the runtime services table so that we can grab the physical 391 * address of several of the EFI runtime functions, needed to 392 * set the firmware into virtual mode. 393 * 394 * When EFI_PARAVIRT is in force then we could not map runtime 395 * service memory region because we do not have direct access to it. 396 * However, runtime services are available through proxy functions 397 * (e.g. in case of Xen dom0 EFI implementation they call special 398 * hypercall which executes relevant EFI functions) and that is why 399 * they are always enabled. 400 */ 401 402 if (!efi_enabled(EFI_PARAVIRT)) { 403 if (efi_enabled(EFI_64BIT)) 404 rv = efi_runtime_init64(); 405 else 406 rv = efi_runtime_init32(); 407 408 if (rv) 409 return rv; 410 } 411 412 set_bit(EFI_RUNTIME_SERVICES, &efi.flags); 413 414 return 0; 415} 416 417static int __init efi_memmap_init(void) 418{ 419 if (efi_enabled(EFI_PARAVIRT)) 420 return 0; 421 422 /* Map the EFI memory map */ 423 memmap.map = early_memremap((unsigned long)memmap.phys_map, 424 memmap.nr_map * memmap.desc_size); 425 if (memmap.map == NULL) { 426 pr_err("Could not map the memory map!\n"); 427 return -ENOMEM; 428 } 429 memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size); 430 431 if (add_efi_memmap) 432 do_add_efi_memmap(); 433 434 set_bit(EFI_MEMMAP, &efi.flags); 435 436 return 0; 437} 438 439void __init efi_init(void) 440{ 441 efi_char16_t *c16; 442 char vendor[100] = "unknown"; 443 int i = 0; 444 void *tmp; 445 446#ifdef CONFIG_X86_32 447 if (boot_params.efi_info.efi_systab_hi || 448 boot_params.efi_info.efi_memmap_hi) { 449 pr_info("Table located above 4GB, disabling EFI.\n"); 450 return; 451 } 452 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab; 453#else 454 efi_phys.systab = (efi_system_table_t *) 455 (boot_params.efi_info.efi_systab | 456 ((__u64)boot_params.efi_info.efi_systab_hi<<32)); 457#endif 458 459 if (efi_systab_init(efi_phys.systab)) 460 return; 461 462 efi.config_table = (unsigned long)efi.systab->tables; 463 efi.fw_vendor = (unsigned long)efi.systab->fw_vendor; 464 efi.runtime = (unsigned long)efi.systab->runtime; 465 466 /* 467 * Show what we know for posterity 468 */ 469 c16 = tmp = early_memremap(efi.systab->fw_vendor, 2); 470 if (c16) { 471 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i) 472 vendor[i] = *c16++; 473 vendor[i] = '\0'; 474 } else 475 pr_err("Could not map the firmware vendor!\n"); 476 early_memunmap(tmp, 2); 477 478 pr_info("EFI v%u.%.02u by %s\n", 479 efi.systab->hdr.revision >> 16, 480 efi.systab->hdr.revision & 0xffff, vendor); 481 482 if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables)) 483 return; 484 485 if (efi_config_init(arch_tables)) 486 return; 487 488 /* 489 * Note: We currently don't support runtime services on an EFI 490 * that doesn't match the kernel 32/64-bit mode. 491 */ 492 493 if (!efi_runtime_supported()) 494 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n"); 495 else { 496 if (efi_runtime_disabled() || efi_runtime_init()) 497 return; 498 } 499 if (efi_memmap_init()) 500 return; 501 502 if (efi_enabled(EFI_DBG)) 503 print_efi_memmap(); 504} 505 506void __init efi_late_init(void) 507{ 508 efi_bgrt_init(); 509} 510 511void __init efi_set_executable(efi_memory_desc_t *md, bool executable) 512{ 513 u64 addr, npages; 514 515 addr = md->virt_addr; 516 npages = md->num_pages; 517 518 memrange_efi_to_native(&addr, &npages); 519 520 if (executable) 521 set_memory_x(addr, npages); 522 else 523 set_memory_nx(addr, npages); 524} 525 526void __init runtime_code_page_mkexec(void) 527{ 528 efi_memory_desc_t *md; 529 void *p; 530 531 /* Make EFI runtime service code area executable */ 532 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 533 md = p; 534 535 if (md->type != EFI_RUNTIME_SERVICES_CODE) 536 continue; 537 538 efi_set_executable(md, true); 539 } 540} 541 542void __init efi_memory_uc(u64 addr, unsigned long size) 543{ 544 unsigned long page_shift = 1UL << EFI_PAGE_SHIFT; 545 u64 npages; 546 547 npages = round_up(size, page_shift) / page_shift; 548 memrange_efi_to_native(&addr, &npages); 549 set_memory_uc(addr, npages); 550} 551 552void __init old_map_region(efi_memory_desc_t *md) 553{ 554 u64 start_pfn, end_pfn, end; 555 unsigned long size; 556 void *va; 557 558 start_pfn = PFN_DOWN(md->phys_addr); 559 size = md->num_pages << PAGE_SHIFT; 560 end = md->phys_addr + size; 561 end_pfn = PFN_UP(end); 562 563 if (pfn_range_is_mapped(start_pfn, end_pfn)) { 564 va = __va(md->phys_addr); 565 566 if (!(md->attribute & EFI_MEMORY_WB)) 567 efi_memory_uc((u64)(unsigned long)va, size); 568 } else 569 va = efi_ioremap(md->phys_addr, size, 570 md->type, md->attribute); 571 572 md->virt_addr = (u64) (unsigned long) va; 573 if (!va) 574 pr_err("ioremap of 0x%llX failed!\n", 575 (unsigned long long)md->phys_addr); 576} 577 578/* Merge contiguous regions of the same type and attribute */ 579static void __init efi_merge_regions(void) 580{ 581 void *p; 582 efi_memory_desc_t *md, *prev_md = NULL; 583 584 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 585 u64 prev_size; 586 md = p; 587 588 if (!prev_md) { 589 prev_md = md; 590 continue; 591 } 592 593 if (prev_md->type != md->type || 594 prev_md->attribute != md->attribute) { 595 prev_md = md; 596 continue; 597 } 598 599 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT; 600 601 if (md->phys_addr == (prev_md->phys_addr + prev_size)) { 602 prev_md->num_pages += md->num_pages; 603 md->type = EFI_RESERVED_TYPE; 604 md->attribute = 0; 605 continue; 606 } 607 prev_md = md; 608 } 609} 610 611static void __init get_systab_virt_addr(efi_memory_desc_t *md) 612{ 613 unsigned long size; 614 u64 end, systab; 615 616 size = md->num_pages << EFI_PAGE_SHIFT; 617 end = md->phys_addr + size; 618 systab = (u64)(unsigned long)efi_phys.systab; 619 if (md->phys_addr <= systab && systab < end) { 620 systab += md->virt_addr - md->phys_addr; 621 efi.systab = (efi_system_table_t *)(unsigned long)systab; 622 } 623} 624 625static void __init save_runtime_map(void) 626{ 627#ifdef CONFIG_KEXEC 628 efi_memory_desc_t *md; 629 void *tmp, *p, *q = NULL; 630 int count = 0; 631 632 if (efi_enabled(EFI_OLD_MEMMAP)) 633 return; 634 635 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 636 md = p; 637 638 if (!(md->attribute & EFI_MEMORY_RUNTIME) || 639 (md->type == EFI_BOOT_SERVICES_CODE) || 640 (md->type == EFI_BOOT_SERVICES_DATA)) 641 continue; 642 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL); 643 if (!tmp) 644 goto out; 645 q = tmp; 646 647 memcpy(q + count * memmap.desc_size, md, memmap.desc_size); 648 count++; 649 } 650 651 efi_runtime_map_setup(q, count, memmap.desc_size); 652 return; 653 654out: 655 kfree(q); 656 pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n"); 657#endif 658} 659 660static void *realloc_pages(void *old_memmap, int old_shift) 661{ 662 void *ret; 663 664 ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1); 665 if (!ret) 666 goto out; 667 668 /* 669 * A first-time allocation doesn't have anything to copy. 670 */ 671 if (!old_memmap) 672 return ret; 673 674 memcpy(ret, old_memmap, PAGE_SIZE << old_shift); 675 676out: 677 free_pages((unsigned long)old_memmap, old_shift); 678 return ret; 679} 680 681/* 682 * Iterate the EFI memory map in reverse order because the regions 683 * will be mapped top-down. The end result is the same as if we had 684 * mapped things forward, but doesn't require us to change the 685 * existing implementation of efi_map_region(). 686 */ 687static inline void *efi_map_next_entry_reverse(void *entry) 688{ 689 /* Initial call */ 690 if (!entry) 691 return memmap.map_end - memmap.desc_size; 692 693 entry -= memmap.desc_size; 694 if (entry < memmap.map) 695 return NULL; 696 697 return entry; 698} 699 700/* 701 * efi_map_next_entry - Return the next EFI memory map descriptor 702 * @entry: Previous EFI memory map descriptor 703 * 704 * This is a helper function to iterate over the EFI memory map, which 705 * we do in different orders depending on the current configuration. 706 * 707 * To begin traversing the memory map @entry must be %NULL. 708 * 709 * Returns %NULL when we reach the end of the memory map. 710 */ 711static void *efi_map_next_entry(void *entry) 712{ 713 if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) { 714 /* 715 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE 716 * config table feature requires us to map all entries 717 * in the same order as they appear in the EFI memory 718 * map. That is to say, entry N must have a lower 719 * virtual address than entry N+1. This is because the 720 * firmware toolchain leaves relative references in 721 * the code/data sections, which are split and become 722 * separate EFI memory regions. Mapping things 723 * out-of-order leads to the firmware accessing 724 * unmapped addresses. 725 * 726 * Since we need to map things this way whether or not 727 * the kernel actually makes use of 728 * EFI_PROPERTIES_TABLE, let's just switch to this 729 * scheme by default for 64-bit. 730 */ 731 return efi_map_next_entry_reverse(entry); 732 } 733 734 /* Initial call */ 735 if (!entry) 736 return memmap.map; 737 738 entry += memmap.desc_size; 739 if (entry >= memmap.map_end) 740 return NULL; 741 742 return entry; 743} 744 745/* 746 * Map the efi memory ranges of the runtime services and update new_mmap with 747 * virtual addresses. 748 */ 749static void * __init efi_map_regions(int *count, int *pg_shift) 750{ 751 void *p, *new_memmap = NULL; 752 unsigned long left = 0; 753 efi_memory_desc_t *md; 754 755 p = NULL; 756 while ((p = efi_map_next_entry(p))) { 757 md = p; 758 if (!(md->attribute & EFI_MEMORY_RUNTIME)) { 759#ifdef CONFIG_X86_64 760 if (md->type != EFI_BOOT_SERVICES_CODE && 761 md->type != EFI_BOOT_SERVICES_DATA) 762#endif 763 continue; 764 } 765 766 efi_map_region(md); 767 get_systab_virt_addr(md); 768 769 if (left < memmap.desc_size) { 770 new_memmap = realloc_pages(new_memmap, *pg_shift); 771 if (!new_memmap) 772 return NULL; 773 774 left += PAGE_SIZE << *pg_shift; 775 (*pg_shift)++; 776 } 777 778 memcpy(new_memmap + (*count * memmap.desc_size), md, 779 memmap.desc_size); 780 781 left -= memmap.desc_size; 782 (*count)++; 783 } 784 785 return new_memmap; 786} 787 788static void __init kexec_enter_virtual_mode(void) 789{ 790#ifdef CONFIG_KEXEC 791 efi_memory_desc_t *md; 792 void *p; 793 794 efi.systab = NULL; 795 796 /* 797 * We don't do virtual mode, since we don't do runtime services, on 798 * non-native EFI 799 */ 800 if (!efi_is_native()) { 801 efi_unmap_memmap(); 802 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 803 return; 804 } 805 806 /* 807 * Map efi regions which were passed via setup_data. The virt_addr is a 808 * fixed addr which was used in first kernel of a kexec boot. 809 */ 810 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 811 md = p; 812 efi_map_region_fixed(md); /* FIXME: add error handling */ 813 get_systab_virt_addr(md); 814 } 815 816 save_runtime_map(); 817 818 BUG_ON(!efi.systab); 819 820 efi_sync_low_kernel_mappings(); 821 822 /* 823 * Now that EFI is in virtual mode, update the function 824 * pointers in the runtime service table to the new virtual addresses. 825 * 826 * Call EFI services through wrapper functions. 827 */ 828 efi.runtime_version = efi_systab.hdr.revision; 829 830 efi_native_runtime_setup(); 831 832 efi.set_virtual_address_map = NULL; 833 834 if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX)) 835 runtime_code_page_mkexec(); 836 837 /* clean DUMMY object */ 838 efi_delete_dummy_variable(); 839#endif 840} 841 842/* 843 * This function will switch the EFI runtime services to virtual mode. 844 * Essentially, we look through the EFI memmap and map every region that 845 * has the runtime attribute bit set in its memory descriptor into the 846 * ->trampoline_pgd page table using a top-down VA allocation scheme. 847 * 848 * The old method which used to update that memory descriptor with the 849 * virtual address obtained from ioremap() is still supported when the 850 * kernel is booted with efi=old_map on its command line. Same old 851 * method enabled the runtime services to be called without having to 852 * thunk back into physical mode for every invocation. 853 * 854 * The new method does a pagetable switch in a preemption-safe manner 855 * so that we're in a different address space when calling a runtime 856 * function. For function arguments passing we do copy the PGDs of the 857 * kernel page table into ->trampoline_pgd prior to each call. 858 * 859 * Specially for kexec boot, efi runtime maps in previous kernel should 860 * be passed in via setup_data. In that case runtime ranges will be mapped 861 * to the same virtual addresses as the first kernel, see 862 * kexec_enter_virtual_mode(). 863 */ 864static void __init __efi_enter_virtual_mode(void) 865{ 866 int count = 0, pg_shift = 0; 867 void *new_memmap = NULL; 868 efi_status_t status; 869 870 efi.systab = NULL; 871 872 efi_merge_regions(); 873 new_memmap = efi_map_regions(&count, &pg_shift); 874 if (!new_memmap) { 875 pr_err("Error reallocating memory, EFI runtime non-functional!\n"); 876 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 877 return; 878 } 879 880 save_runtime_map(); 881 882 BUG_ON(!efi.systab); 883 884 if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) { 885 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 886 return; 887 } 888 889 efi_sync_low_kernel_mappings(); 890 efi_dump_pagetable(); 891 892 if (efi_is_native()) { 893 status = phys_efi_set_virtual_address_map( 894 memmap.desc_size * count, 895 memmap.desc_size, 896 memmap.desc_version, 897 (efi_memory_desc_t *)__pa(new_memmap)); 898 } else { 899 status = efi_thunk_set_virtual_address_map( 900 efi_phys.set_virtual_address_map, 901 memmap.desc_size * count, 902 memmap.desc_size, 903 memmap.desc_version, 904 (efi_memory_desc_t *)__pa(new_memmap)); 905 } 906 907 if (status != EFI_SUCCESS) { 908 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n", 909 status); 910 panic("EFI call to SetVirtualAddressMap() failed!"); 911 } 912 913 /* 914 * Now that EFI is in virtual mode, update the function 915 * pointers in the runtime service table to the new virtual addresses. 916 * 917 * Call EFI services through wrapper functions. 918 */ 919 efi.runtime_version = efi_systab.hdr.revision; 920 921 if (efi_is_native()) 922 efi_native_runtime_setup(); 923 else 924 efi_thunk_runtime_setup(); 925 926 efi.set_virtual_address_map = NULL; 927 928 efi_runtime_mkexec(); 929 930 /* 931 * We mapped the descriptor array into the EFI pagetable above but we're 932 * not unmapping it here. Here's why: 933 * 934 * We're copying select PGDs from the kernel page table to the EFI page 935 * table and when we do so and make changes to those PGDs like unmapping 936 * stuff from them, those changes appear in the kernel page table and we 937 * go boom. 938 * 939 * From setup_real_mode(): 940 * 941 * ... 942 * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd; 943 * 944 * In this particular case, our allocation is in PGD 0 of the EFI page 945 * table but we've copied that PGD from PGD[272] of the EFI page table: 946 * 947 * pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272 948 * 949 * where the direct memory mapping in kernel space is. 950 * 951 * new_memmap's VA comes from that direct mapping and thus clearing it, 952 * it would get cleared in the kernel page table too. 953 * 954 * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift); 955 */ 956 free_pages((unsigned long)new_memmap, pg_shift); 957 958 /* clean DUMMY object */ 959 efi_delete_dummy_variable(); 960} 961 962void __init efi_enter_virtual_mode(void) 963{ 964 if (efi_enabled(EFI_PARAVIRT)) 965 return; 966 967 if (efi_setup) 968 kexec_enter_virtual_mode(); 969 else 970 __efi_enter_virtual_mode(); 971} 972 973/* 974 * Convenience functions to obtain memory types and attributes 975 */ 976u32 efi_mem_type(unsigned long phys_addr) 977{ 978 efi_memory_desc_t *md; 979 void *p; 980 981 if (!efi_enabled(EFI_MEMMAP)) 982 return 0; 983 984 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 985 md = p; 986 if ((md->phys_addr <= phys_addr) && 987 (phys_addr < (md->phys_addr + 988 (md->num_pages << EFI_PAGE_SHIFT)))) 989 return md->type; 990 } 991 return 0; 992} 993 994u64 efi_mem_attributes(unsigned long phys_addr) 995{ 996 efi_memory_desc_t *md; 997 void *p; 998 999 if (!efi_enabled(EFI_MEMMAP)) 1000 return 0; 1001 1002 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 1003 md = p; 1004 if ((md->phys_addr <= phys_addr) && 1005 (phys_addr < (md->phys_addr + 1006 (md->num_pages << EFI_PAGE_SHIFT)))) 1007 return md->attribute; 1008 } 1009 return 0; 1010} 1011 1012static int __init arch_parse_efi_cmdline(char *str) 1013{ 1014 if (!str) { 1015 pr_warn("need at least one option\n"); 1016 return -EINVAL; 1017 } 1018 1019 if (parse_option_str(str, "old_map")) 1020 set_bit(EFI_OLD_MEMMAP, &efi.flags); 1021 if (parse_option_str(str, "debug")) 1022 set_bit(EFI_DBG, &efi.flags); 1023 1024 return 0; 1025} 1026early_param("efi", arch_parse_efi_cmdline); 1027