1/****************************************************************************** 2 * grant_table.c 3 * 4 * Granting foreign access to our memory reservation. 5 * 6 * Copyright (c) 2005-2006, Christopher Clark 7 * Copyright (c) 2004-2005, K A Fraser 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License version 2 11 * as published by the Free Software Foundation; or, when distributed 12 * separately from the Linux kernel or incorporated into other 13 * software packages, subject to the following license: 14 * 15 * Permission is hereby granted, free of charge, to any person obtaining a copy 16 * of this source file (the "Software"), to deal in the Software without 17 * restriction, including without limitation the rights to use, copy, modify, 18 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 19 * and to permit persons to whom the Software is furnished to do so, subject to 20 * the following conditions: 21 * 22 * The above copyright notice and this permission notice shall be included in 23 * all copies or substantial portions of the Software. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 31 * IN THE SOFTWARE. 32 */ 33 34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 35 36#include <linux/module.h> 37#include <linux/sched.h> 38#include <linux/mm.h> 39#include <linux/slab.h> 40#include <linux/vmalloc.h> 41#include <linux/uaccess.h> 42#include <linux/io.h> 43#include <linux/delay.h> 44#include <linux/hardirq.h> 45#include <linux/workqueue.h> 46 47#include <xen/xen.h> 48#include <xen/interface/xen.h> 49#include <xen/page.h> 50#include <xen/grant_table.h> 51#include <xen/interface/memory.h> 52#include <xen/hvc-console.h> 53#include <xen/swiotlb-xen.h> 54#include <xen/balloon.h> 55#include <asm/xen/hypercall.h> 56#include <asm/xen/interface.h> 57 58#include <asm/pgtable.h> 59#include <asm/sync_bitops.h> 60 61/* External tools reserve first few grant table entries. */ 62#define NR_RESERVED_ENTRIES 8 63#define GNTTAB_LIST_END 0xffffffff 64 65static grant_ref_t **gnttab_list; 66static unsigned int nr_grant_frames; 67static int gnttab_free_count; 68static grant_ref_t gnttab_free_head; 69static DEFINE_SPINLOCK(gnttab_list_lock); 70struct grant_frames xen_auto_xlat_grant_frames; 71 72static union { 73 struct grant_entry_v1 *v1; 74 void *addr; 75} gnttab_shared; 76 77/*This is a structure of function pointers for grant table*/ 78struct gnttab_ops { 79 /* 80 * Mapping a list of frames for storing grant entries. Frames parameter 81 * is used to store grant table address when grant table being setup, 82 * nr_gframes is the number of frames to map grant table. Returning 83 * GNTST_okay means success and negative value means failure. 84 */ 85 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes); 86 /* 87 * Release a list of frames which are mapped in map_frames for grant 88 * entry status. 89 */ 90 void (*unmap_frames)(void); 91 /* 92 * Introducing a valid entry into the grant table, granting the frame of 93 * this grant entry to domain for accessing or transfering. Ref 94 * parameter is reference of this introduced grant entry, domid is id of 95 * granted domain, frame is the page frame to be granted, and flags is 96 * status of the grant entry to be updated. 97 */ 98 void (*update_entry)(grant_ref_t ref, domid_t domid, 99 unsigned long frame, unsigned flags); 100 /* 101 * Stop granting a grant entry to domain for accessing. Ref parameter is 102 * reference of a grant entry whose grant access will be stopped, 103 * readonly is not in use in this function. If the grant entry is 104 * currently mapped for reading or writing, just return failure(==0) 105 * directly and don't tear down the grant access. Otherwise, stop grant 106 * access for this entry and return success(==1). 107 */ 108 int (*end_foreign_access_ref)(grant_ref_t ref, int readonly); 109 /* 110 * Stop granting a grant entry to domain for transfer. Ref parameter is 111 * reference of a grant entry whose grant transfer will be stopped. If 112 * tranfer has not started, just reclaim the grant entry and return 113 * failure(==0). Otherwise, wait for the transfer to complete and then 114 * return the frame. 115 */ 116 unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref); 117 /* 118 * Query the status of a grant entry. Ref parameter is reference of 119 * queried grant entry, return value is the status of queried entry. 120 * Detailed status(writing/reading) can be gotten from the return value 121 * by bit operations. 122 */ 123 int (*query_foreign_access)(grant_ref_t ref); 124}; 125 126struct unmap_refs_callback_data { 127 struct completion completion; 128 int result; 129}; 130 131static struct gnttab_ops *gnttab_interface; 132 133static int grant_table_version; 134static int grefs_per_grant_frame; 135 136static struct gnttab_free_callback *gnttab_free_callback_list; 137 138static int gnttab_expand(unsigned int req_entries); 139 140#define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 141#define SPP (PAGE_SIZE / sizeof(grant_status_t)) 142 143static inline grant_ref_t *__gnttab_entry(grant_ref_t entry) 144{ 145 return &gnttab_list[(entry) / RPP][(entry) % RPP]; 146} 147/* This can be used as an l-value */ 148#define gnttab_entry(entry) (*__gnttab_entry(entry)) 149 150static int get_free_entries(unsigned count) 151{ 152 unsigned long flags; 153 int ref, rc = 0; 154 grant_ref_t head; 155 156 spin_lock_irqsave(&gnttab_list_lock, flags); 157 158 if ((gnttab_free_count < count) && 159 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 160 spin_unlock_irqrestore(&gnttab_list_lock, flags); 161 return rc; 162 } 163 164 ref = head = gnttab_free_head; 165 gnttab_free_count -= count; 166 while (count-- > 1) 167 head = gnttab_entry(head); 168 gnttab_free_head = gnttab_entry(head); 169 gnttab_entry(head) = GNTTAB_LIST_END; 170 171 spin_unlock_irqrestore(&gnttab_list_lock, flags); 172 173 return ref; 174} 175 176static void do_free_callbacks(void) 177{ 178 struct gnttab_free_callback *callback, *next; 179 180 callback = gnttab_free_callback_list; 181 gnttab_free_callback_list = NULL; 182 183 while (callback != NULL) { 184 next = callback->next; 185 if (gnttab_free_count >= callback->count) { 186 callback->next = NULL; 187 callback->fn(callback->arg); 188 } else { 189 callback->next = gnttab_free_callback_list; 190 gnttab_free_callback_list = callback; 191 } 192 callback = next; 193 } 194} 195 196static inline void check_free_callbacks(void) 197{ 198 if (unlikely(gnttab_free_callback_list)) 199 do_free_callbacks(); 200} 201 202static void put_free_entry(grant_ref_t ref) 203{ 204 unsigned long flags; 205 spin_lock_irqsave(&gnttab_list_lock, flags); 206 gnttab_entry(ref) = gnttab_free_head; 207 gnttab_free_head = ref; 208 gnttab_free_count++; 209 check_free_callbacks(); 210 spin_unlock_irqrestore(&gnttab_list_lock, flags); 211} 212 213/* 214 * Following applies to gnttab_update_entry_v1. 215 * Introducing a valid entry into the grant table: 216 * 1. Write ent->domid. 217 * 2. Write ent->frame: 218 * GTF_permit_access: Frame to which access is permitted. 219 * GTF_accept_transfer: Pseudo-phys frame slot being filled by new 220 * frame, or zero if none. 221 * 3. Write memory barrier (WMB). 222 * 4. Write ent->flags, inc. valid type. 223 */ 224static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid, 225 unsigned long frame, unsigned flags) 226{ 227 gnttab_shared.v1[ref].domid = domid; 228 gnttab_shared.v1[ref].frame = frame; 229 wmb(); 230 gnttab_shared.v1[ref].flags = flags; 231} 232 233/* 234 * Public grant-issuing interface functions 235 */ 236void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 237 unsigned long frame, int readonly) 238{ 239 gnttab_interface->update_entry(ref, domid, frame, 240 GTF_permit_access | (readonly ? GTF_readonly : 0)); 241} 242EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref); 243 244int gnttab_grant_foreign_access(domid_t domid, unsigned long frame, 245 int readonly) 246{ 247 int ref; 248 249 ref = get_free_entries(1); 250 if (unlikely(ref < 0)) 251 return -ENOSPC; 252 253 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly); 254 255 return ref; 256} 257EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access); 258 259static int gnttab_query_foreign_access_v1(grant_ref_t ref) 260{ 261 return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing); 262} 263 264int gnttab_query_foreign_access(grant_ref_t ref) 265{ 266 return gnttab_interface->query_foreign_access(ref); 267} 268EXPORT_SYMBOL_GPL(gnttab_query_foreign_access); 269 270static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly) 271{ 272 u16 flags, nflags; 273 u16 *pflags; 274 275 pflags = &gnttab_shared.v1[ref].flags; 276 nflags = *pflags; 277 do { 278 flags = nflags; 279 if (flags & (GTF_reading|GTF_writing)) 280 return 0; 281 } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags); 282 283 return 1; 284} 285 286static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 287{ 288 return gnttab_interface->end_foreign_access_ref(ref, readonly); 289} 290 291int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 292{ 293 if (_gnttab_end_foreign_access_ref(ref, readonly)) 294 return 1; 295 pr_warn("WARNING: g.e. %#x still in use!\n", ref); 296 return 0; 297} 298EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref); 299 300struct deferred_entry { 301 struct list_head list; 302 grant_ref_t ref; 303 bool ro; 304 uint16_t warn_delay; 305 struct page *page; 306}; 307static LIST_HEAD(deferred_list); 308static void gnttab_handle_deferred(unsigned long); 309static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0); 310 311static void gnttab_handle_deferred(unsigned long unused) 312{ 313 unsigned int nr = 10; 314 struct deferred_entry *first = NULL; 315 unsigned long flags; 316 317 spin_lock_irqsave(&gnttab_list_lock, flags); 318 while (nr--) { 319 struct deferred_entry *entry 320 = list_first_entry(&deferred_list, 321 struct deferred_entry, list); 322 323 if (entry == first) 324 break; 325 list_del(&entry->list); 326 spin_unlock_irqrestore(&gnttab_list_lock, flags); 327 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) { 328 put_free_entry(entry->ref); 329 if (entry->page) { 330 pr_debug("freeing g.e. %#x (pfn %#lx)\n", 331 entry->ref, page_to_pfn(entry->page)); 332 __free_page(entry->page); 333 } else 334 pr_info("freeing g.e. %#x\n", entry->ref); 335 kfree(entry); 336 entry = NULL; 337 } else { 338 if (!--entry->warn_delay) 339 pr_info("g.e. %#x still pending\n", entry->ref); 340 if (!first) 341 first = entry; 342 } 343 spin_lock_irqsave(&gnttab_list_lock, flags); 344 if (entry) 345 list_add_tail(&entry->list, &deferred_list); 346 else if (list_empty(&deferred_list)) 347 break; 348 } 349 if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) { 350 deferred_timer.expires = jiffies + HZ; 351 add_timer(&deferred_timer); 352 } 353 spin_unlock_irqrestore(&gnttab_list_lock, flags); 354} 355 356static void gnttab_add_deferred(grant_ref_t ref, bool readonly, 357 struct page *page) 358{ 359 struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC); 360 const char *what = KERN_WARNING "leaking"; 361 362 if (entry) { 363 unsigned long flags; 364 365 entry->ref = ref; 366 entry->ro = readonly; 367 entry->page = page; 368 entry->warn_delay = 60; 369 spin_lock_irqsave(&gnttab_list_lock, flags); 370 list_add_tail(&entry->list, &deferred_list); 371 if (!timer_pending(&deferred_timer)) { 372 deferred_timer.expires = jiffies + HZ; 373 add_timer(&deferred_timer); 374 } 375 spin_unlock_irqrestore(&gnttab_list_lock, flags); 376 what = KERN_DEBUG "deferring"; 377 } 378 printk("%s g.e. %#x (pfn %#lx)\n", 379 what, ref, page ? page_to_pfn(page) : -1); 380} 381 382void gnttab_end_foreign_access(grant_ref_t ref, int readonly, 383 unsigned long page) 384{ 385 if (gnttab_end_foreign_access_ref(ref, readonly)) { 386 put_free_entry(ref); 387 if (page != 0) 388 free_page(page); 389 } else 390 gnttab_add_deferred(ref, readonly, 391 page ? virt_to_page(page) : NULL); 392} 393EXPORT_SYMBOL_GPL(gnttab_end_foreign_access); 394 395int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn) 396{ 397 int ref; 398 399 ref = get_free_entries(1); 400 if (unlikely(ref < 0)) 401 return -ENOSPC; 402 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 403 404 return ref; 405} 406EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer); 407 408void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 409 unsigned long pfn) 410{ 411 gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer); 412} 413EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref); 414 415static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref) 416{ 417 unsigned long frame; 418 u16 flags; 419 u16 *pflags; 420 421 pflags = &gnttab_shared.v1[ref].flags; 422 423 /* 424 * If a transfer is not even yet started, try to reclaim the grant 425 * reference and return failure (== 0). 426 */ 427 while (!((flags = *pflags) & GTF_transfer_committed)) { 428 if (sync_cmpxchg(pflags, flags, 0) == flags) 429 return 0; 430 cpu_relax(); 431 } 432 433 /* If a transfer is in progress then wait until it is completed. */ 434 while (!(flags & GTF_transfer_completed)) { 435 flags = *pflags; 436 cpu_relax(); 437 } 438 439 rmb(); /* Read the frame number /after/ reading completion status. */ 440 frame = gnttab_shared.v1[ref].frame; 441 BUG_ON(frame == 0); 442 443 return frame; 444} 445 446unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref) 447{ 448 return gnttab_interface->end_foreign_transfer_ref(ref); 449} 450EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref); 451 452unsigned long gnttab_end_foreign_transfer(grant_ref_t ref) 453{ 454 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 455 put_free_entry(ref); 456 return frame; 457} 458EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer); 459 460void gnttab_free_grant_reference(grant_ref_t ref) 461{ 462 put_free_entry(ref); 463} 464EXPORT_SYMBOL_GPL(gnttab_free_grant_reference); 465 466void gnttab_free_grant_references(grant_ref_t head) 467{ 468 grant_ref_t ref; 469 unsigned long flags; 470 int count = 1; 471 if (head == GNTTAB_LIST_END) 472 return; 473 spin_lock_irqsave(&gnttab_list_lock, flags); 474 ref = head; 475 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 476 ref = gnttab_entry(ref); 477 count++; 478 } 479 gnttab_entry(ref) = gnttab_free_head; 480 gnttab_free_head = head; 481 gnttab_free_count += count; 482 check_free_callbacks(); 483 spin_unlock_irqrestore(&gnttab_list_lock, flags); 484} 485EXPORT_SYMBOL_GPL(gnttab_free_grant_references); 486 487int gnttab_alloc_grant_references(u16 count, grant_ref_t *head) 488{ 489 int h = get_free_entries(count); 490 491 if (h < 0) 492 return -ENOSPC; 493 494 *head = h; 495 496 return 0; 497} 498EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references); 499 500int gnttab_empty_grant_references(const grant_ref_t *private_head) 501{ 502 return (*private_head == GNTTAB_LIST_END); 503} 504EXPORT_SYMBOL_GPL(gnttab_empty_grant_references); 505 506int gnttab_claim_grant_reference(grant_ref_t *private_head) 507{ 508 grant_ref_t g = *private_head; 509 if (unlikely(g == GNTTAB_LIST_END)) 510 return -ENOSPC; 511 *private_head = gnttab_entry(g); 512 return g; 513} 514EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference); 515 516void gnttab_release_grant_reference(grant_ref_t *private_head, 517 grant_ref_t release) 518{ 519 gnttab_entry(release) = *private_head; 520 *private_head = release; 521} 522EXPORT_SYMBOL_GPL(gnttab_release_grant_reference); 523 524void gnttab_request_free_callback(struct gnttab_free_callback *callback, 525 void (*fn)(void *), void *arg, u16 count) 526{ 527 unsigned long flags; 528 struct gnttab_free_callback *cb; 529 530 spin_lock_irqsave(&gnttab_list_lock, flags); 531 532 /* Check if the callback is already on the list */ 533 cb = gnttab_free_callback_list; 534 while (cb) { 535 if (cb == callback) 536 goto out; 537 cb = cb->next; 538 } 539 540 callback->fn = fn; 541 callback->arg = arg; 542 callback->count = count; 543 callback->next = gnttab_free_callback_list; 544 gnttab_free_callback_list = callback; 545 check_free_callbacks(); 546out: 547 spin_unlock_irqrestore(&gnttab_list_lock, flags); 548} 549EXPORT_SYMBOL_GPL(gnttab_request_free_callback); 550 551void gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 552{ 553 struct gnttab_free_callback **pcb; 554 unsigned long flags; 555 556 spin_lock_irqsave(&gnttab_list_lock, flags); 557 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 558 if (*pcb == callback) { 559 *pcb = callback->next; 560 break; 561 } 562 } 563 spin_unlock_irqrestore(&gnttab_list_lock, flags); 564} 565EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback); 566 567static int grow_gnttab_list(unsigned int more_frames) 568{ 569 unsigned int new_nr_grant_frames, extra_entries, i; 570 unsigned int nr_glist_frames, new_nr_glist_frames; 571 572 BUG_ON(grefs_per_grant_frame == 0); 573 574 new_nr_grant_frames = nr_grant_frames + more_frames; 575 extra_entries = more_frames * grefs_per_grant_frame; 576 577 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP; 578 new_nr_glist_frames = 579 (new_nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP; 580 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) { 581 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC); 582 if (!gnttab_list[i]) 583 goto grow_nomem; 584 } 585 586 587 for (i = grefs_per_grant_frame * nr_grant_frames; 588 i < grefs_per_grant_frame * new_nr_grant_frames - 1; i++) 589 gnttab_entry(i) = i + 1; 590 591 gnttab_entry(i) = gnttab_free_head; 592 gnttab_free_head = grefs_per_grant_frame * nr_grant_frames; 593 gnttab_free_count += extra_entries; 594 595 nr_grant_frames = new_nr_grant_frames; 596 597 check_free_callbacks(); 598 599 return 0; 600 601grow_nomem: 602 while (i-- > nr_glist_frames) 603 free_page((unsigned long) gnttab_list[i]); 604 return -ENOMEM; 605} 606 607static unsigned int __max_nr_grant_frames(void) 608{ 609 struct gnttab_query_size query; 610 int rc; 611 612 query.dom = DOMID_SELF; 613 614 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 615 if ((rc < 0) || (query.status != GNTST_okay)) 616 return 4; /* Legacy max supported number of frames */ 617 618 return query.max_nr_frames; 619} 620 621unsigned int gnttab_max_grant_frames(void) 622{ 623 unsigned int xen_max = __max_nr_grant_frames(); 624 static unsigned int boot_max_nr_grant_frames; 625 626 /* First time, initialize it properly. */ 627 if (!boot_max_nr_grant_frames) 628 boot_max_nr_grant_frames = __max_nr_grant_frames(); 629 630 if (xen_max > boot_max_nr_grant_frames) 631 return boot_max_nr_grant_frames; 632 return xen_max; 633} 634EXPORT_SYMBOL_GPL(gnttab_max_grant_frames); 635 636int gnttab_setup_auto_xlat_frames(phys_addr_t addr) 637{ 638 xen_pfn_t *pfn; 639 unsigned int max_nr_gframes = __max_nr_grant_frames(); 640 unsigned int i; 641 void *vaddr; 642 643 if (xen_auto_xlat_grant_frames.count) 644 return -EINVAL; 645 646 vaddr = xen_remap(addr, PAGE_SIZE * max_nr_gframes); 647 if (vaddr == NULL) { 648 pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n", 649 &addr); 650 return -ENOMEM; 651 } 652 pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL); 653 if (!pfn) { 654 xen_unmap(vaddr); 655 return -ENOMEM; 656 } 657 for (i = 0; i < max_nr_gframes; i++) 658 pfn[i] = PFN_DOWN(addr) + i; 659 660 xen_auto_xlat_grant_frames.vaddr = vaddr; 661 xen_auto_xlat_grant_frames.pfn = pfn; 662 xen_auto_xlat_grant_frames.count = max_nr_gframes; 663 664 return 0; 665} 666EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames); 667 668void gnttab_free_auto_xlat_frames(void) 669{ 670 if (!xen_auto_xlat_grant_frames.count) 671 return; 672 kfree(xen_auto_xlat_grant_frames.pfn); 673 xen_unmap(xen_auto_xlat_grant_frames.vaddr); 674 675 xen_auto_xlat_grant_frames.pfn = NULL; 676 xen_auto_xlat_grant_frames.count = 0; 677 xen_auto_xlat_grant_frames.vaddr = NULL; 678} 679EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); 680 681/** 682 * gnttab_alloc_pages - alloc pages suitable for grant mapping into 683 * @nr_pages: number of pages to alloc 684 * @pages: returns the pages 685 */ 686int gnttab_alloc_pages(int nr_pages, struct page **pages) 687{ 688 int i; 689 int ret; 690 691 ret = alloc_xenballooned_pages(nr_pages, pages, false); 692 if (ret < 0) 693 return ret; 694 695 for (i = 0; i < nr_pages; i++) { 696#if BITS_PER_LONG < 64 697 struct xen_page_foreign *foreign; 698 699 foreign = kzalloc(sizeof(*foreign), GFP_KERNEL); 700 if (!foreign) { 701 gnttab_free_pages(nr_pages, pages); 702 return -ENOMEM; 703 } 704 set_page_private(pages[i], (unsigned long)foreign); 705#endif 706 SetPagePrivate(pages[i]); 707 } 708 709 return 0; 710} 711EXPORT_SYMBOL(gnttab_alloc_pages); 712 713/** 714 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages() 715 * @nr_pages; number of pages to free 716 * @pages: the pages 717 */ 718void gnttab_free_pages(int nr_pages, struct page **pages) 719{ 720 int i; 721 722 for (i = 0; i < nr_pages; i++) { 723 if (PagePrivate(pages[i])) { 724#if BITS_PER_LONG < 64 725 kfree((void *)page_private(pages[i])); 726#endif 727 ClearPagePrivate(pages[i]); 728 } 729 } 730 free_xenballooned_pages(nr_pages, pages); 731} 732EXPORT_SYMBOL(gnttab_free_pages); 733 734/* Handling of paged out grant targets (GNTST_eagain) */ 735#define MAX_DELAY 256 736static inline void 737gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status, 738 const char *func) 739{ 740 unsigned delay = 1; 741 742 do { 743 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1)); 744 if (*status == GNTST_eagain) 745 msleep(delay++); 746 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY)); 747 748 if (delay >= MAX_DELAY) { 749 pr_err("%s: %s eagain grant\n", func, current->comm); 750 *status = GNTST_bad_page; 751 } 752} 753 754void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count) 755{ 756 struct gnttab_map_grant_ref *op; 757 758 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count)) 759 BUG(); 760 for (op = batch; op < batch + count; op++) 761 if (op->status == GNTST_eagain) 762 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op, 763 &op->status, __func__); 764} 765EXPORT_SYMBOL_GPL(gnttab_batch_map); 766 767void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count) 768{ 769 struct gnttab_copy *op; 770 771 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count)) 772 BUG(); 773 for (op = batch; op < batch + count; op++) 774 if (op->status == GNTST_eagain) 775 gnttab_retry_eagain_gop(GNTTABOP_copy, op, 776 &op->status, __func__); 777} 778EXPORT_SYMBOL_GPL(gnttab_batch_copy); 779 780int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, 781 struct gnttab_map_grant_ref *kmap_ops, 782 struct page **pages, unsigned int count) 783{ 784 int i, ret; 785 786 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count); 787 if (ret) 788 return ret; 789 790 for (i = 0; i < count; i++) { 791 /* Retry eagain maps */ 792 if (map_ops[i].status == GNTST_eagain) 793 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, 794 &map_ops[i].status, __func__); 795 796 if (map_ops[i].status == GNTST_okay) { 797 struct xen_page_foreign *foreign; 798 799 SetPageForeign(pages[i]); 800 foreign = xen_page_foreign(pages[i]); 801 foreign->domid = map_ops[i].dom; 802 foreign->gref = map_ops[i].ref; 803 } 804 } 805 806 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count); 807} 808EXPORT_SYMBOL_GPL(gnttab_map_refs); 809 810int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, 811 struct gnttab_unmap_grant_ref *kunmap_ops, 812 struct page **pages, unsigned int count) 813{ 814 unsigned int i; 815 int ret; 816 817 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); 818 if (ret) 819 return ret; 820 821 for (i = 0; i < count; i++) 822 ClearPageForeign(pages[i]); 823 824 return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count); 825} 826EXPORT_SYMBOL_GPL(gnttab_unmap_refs); 827 828#define GNTTAB_UNMAP_REFS_DELAY 5 829 830static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item); 831 832static void gnttab_unmap_work(struct work_struct *work) 833{ 834 struct gntab_unmap_queue_data 835 *unmap_data = container_of(work, 836 struct gntab_unmap_queue_data, 837 gnttab_work.work); 838 if (unmap_data->age != UINT_MAX) 839 unmap_data->age++; 840 __gnttab_unmap_refs_async(unmap_data); 841} 842 843static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) 844{ 845 int ret; 846 int pc; 847 848 for (pc = 0; pc < item->count; pc++) { 849 if (page_count(item->pages[pc]) > 1) { 850 unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1); 851 schedule_delayed_work(&item->gnttab_work, 852 msecs_to_jiffies(delay)); 853 return; 854 } 855 } 856 857 ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops, 858 item->pages, item->count); 859 item->done(ret, item); 860} 861 862void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) 863{ 864 INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work); 865 item->age = 0; 866 867 __gnttab_unmap_refs_async(item); 868} 869EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async); 870 871static void unmap_refs_callback(int result, 872 struct gntab_unmap_queue_data *data) 873{ 874 struct unmap_refs_callback_data *d = data->data; 875 876 d->result = result; 877 complete(&d->completion); 878} 879 880int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item) 881{ 882 struct unmap_refs_callback_data data; 883 884 init_completion(&data.completion); 885 item->data = &data; 886 item->done = &unmap_refs_callback; 887 gnttab_unmap_refs_async(item); 888 wait_for_completion(&data.completion); 889 890 return data.result; 891} 892EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync); 893 894static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes) 895{ 896 int rc; 897 898 rc = arch_gnttab_map_shared(frames, nr_gframes, 899 gnttab_max_grant_frames(), 900 &gnttab_shared.addr); 901 BUG_ON(rc); 902 903 return 0; 904} 905 906static void gnttab_unmap_frames_v1(void) 907{ 908 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames); 909} 910 911static int gnttab_map(unsigned int start_idx, unsigned int end_idx) 912{ 913 struct gnttab_setup_table setup; 914 xen_pfn_t *frames; 915 unsigned int nr_gframes = end_idx + 1; 916 int rc; 917 918 if (xen_feature(XENFEAT_auto_translated_physmap)) { 919 struct xen_add_to_physmap xatp; 920 unsigned int i = end_idx; 921 rc = 0; 922 BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes); 923 /* 924 * Loop backwards, so that the first hypercall has the largest 925 * index, ensuring that the table will grow only once. 926 */ 927 do { 928 xatp.domid = DOMID_SELF; 929 xatp.idx = i; 930 xatp.space = XENMAPSPACE_grant_table; 931 xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i]; 932 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp); 933 if (rc != 0) { 934 pr_warn("grant table add_to_physmap failed, err=%d\n", 935 rc); 936 break; 937 } 938 } while (i-- > start_idx); 939 940 return rc; 941 } 942 943 /* No need for kzalloc as it is initialized in following hypercall 944 * GNTTABOP_setup_table. 945 */ 946 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC); 947 if (!frames) 948 return -ENOMEM; 949 950 setup.dom = DOMID_SELF; 951 setup.nr_frames = nr_gframes; 952 set_xen_guest_handle(setup.frame_list, frames); 953 954 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 955 if (rc == -ENOSYS) { 956 kfree(frames); 957 return -ENOSYS; 958 } 959 960 BUG_ON(rc || setup.status); 961 962 rc = gnttab_interface->map_frames(frames, nr_gframes); 963 964 kfree(frames); 965 966 return rc; 967} 968 969static struct gnttab_ops gnttab_v1_ops = { 970 .map_frames = gnttab_map_frames_v1, 971 .unmap_frames = gnttab_unmap_frames_v1, 972 .update_entry = gnttab_update_entry_v1, 973 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1, 974 .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1, 975 .query_foreign_access = gnttab_query_foreign_access_v1, 976}; 977 978static void gnttab_request_version(void) 979{ 980 /* Only version 1 is used, which will always be available. */ 981 grant_table_version = 1; 982 grefs_per_grant_frame = PAGE_SIZE / sizeof(struct grant_entry_v1); 983 gnttab_interface = &gnttab_v1_ops; 984 985 pr_info("Grant tables using version %d layout\n", grant_table_version); 986} 987 988static int gnttab_setup(void) 989{ 990 unsigned int max_nr_gframes; 991 992 max_nr_gframes = gnttab_max_grant_frames(); 993 if (max_nr_gframes < nr_grant_frames) 994 return -ENOSYS; 995 996 if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) { 997 gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr; 998 if (gnttab_shared.addr == NULL) { 999 pr_warn("gnttab share frames (addr=0x%08lx) is not mapped!\n", 1000 (unsigned long)xen_auto_xlat_grant_frames.vaddr); 1001 return -ENOMEM; 1002 } 1003 } 1004 return gnttab_map(0, nr_grant_frames - 1); 1005} 1006 1007int gnttab_resume(void) 1008{ 1009 gnttab_request_version(); 1010 return gnttab_setup(); 1011} 1012 1013int gnttab_suspend(void) 1014{ 1015 if (!xen_feature(XENFEAT_auto_translated_physmap)) 1016 gnttab_interface->unmap_frames(); 1017 return 0; 1018} 1019 1020static int gnttab_expand(unsigned int req_entries) 1021{ 1022 int rc; 1023 unsigned int cur, extra; 1024 1025 BUG_ON(grefs_per_grant_frame == 0); 1026 cur = nr_grant_frames; 1027 extra = ((req_entries + (grefs_per_grant_frame-1)) / 1028 grefs_per_grant_frame); 1029 if (cur + extra > gnttab_max_grant_frames()) 1030 return -ENOSPC; 1031 1032 rc = gnttab_map(cur, cur + extra - 1); 1033 if (rc == 0) 1034 rc = grow_gnttab_list(extra); 1035 1036 return rc; 1037} 1038 1039int gnttab_init(void) 1040{ 1041 int i; 1042 unsigned long max_nr_grant_frames; 1043 unsigned int max_nr_glist_frames, nr_glist_frames; 1044 unsigned int nr_init_grefs; 1045 int ret; 1046 1047 gnttab_request_version(); 1048 max_nr_grant_frames = gnttab_max_grant_frames(); 1049 nr_grant_frames = 1; 1050 1051 /* Determine the maximum number of frames required for the 1052 * grant reference free list on the current hypervisor. 1053 */ 1054 BUG_ON(grefs_per_grant_frame == 0); 1055 max_nr_glist_frames = (max_nr_grant_frames * 1056 grefs_per_grant_frame / RPP); 1057 1058 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *), 1059 GFP_KERNEL); 1060 if (gnttab_list == NULL) 1061 return -ENOMEM; 1062 1063 nr_glist_frames = (nr_grant_frames * grefs_per_grant_frame + RPP - 1) / RPP; 1064 for (i = 0; i < nr_glist_frames; i++) { 1065 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL); 1066 if (gnttab_list[i] == NULL) { 1067 ret = -ENOMEM; 1068 goto ini_nomem; 1069 } 1070 } 1071 1072 ret = arch_gnttab_init(max_nr_grant_frames); 1073 if (ret < 0) 1074 goto ini_nomem; 1075 1076 if (gnttab_setup() < 0) { 1077 ret = -ENODEV; 1078 goto ini_nomem; 1079 } 1080 1081 nr_init_grefs = nr_grant_frames * grefs_per_grant_frame; 1082 1083 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 1084 gnttab_entry(i) = i + 1; 1085 1086 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 1087 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 1088 gnttab_free_head = NR_RESERVED_ENTRIES; 1089 1090 printk("Grant table initialized\n"); 1091 return 0; 1092 1093 ini_nomem: 1094 for (i--; i >= 0; i--) 1095 free_page((unsigned long)gnttab_list[i]); 1096 kfree(gnttab_list); 1097 return ret; 1098} 1099EXPORT_SYMBOL_GPL(gnttab_init); 1100 1101static int __gnttab_init(void) 1102{ 1103 /* Delay grant-table initialization in the PV on HVM case */ 1104 if (xen_hvm_domain()) 1105 return 0; 1106 1107 if (!xen_pv_domain()) 1108 return -ENODEV; 1109 1110 return gnttab_init(); 1111} 1112/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called 1113 * beforehand to initialize xen_auto_xlat_grant_frames. */ 1114core_initcall_sync(__gnttab_init); 1115