1/* 2 * Copyright (C) 2003 Sistina Software Limited. 3 * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8#include "dm-bio-record.h" 9 10#include <linux/init.h> 11#include <linux/mempool.h> 12#include <linux/module.h> 13#include <linux/pagemap.h> 14#include <linux/slab.h> 15#include <linux/workqueue.h> 16#include <linux/device-mapper.h> 17#include <linux/dm-io.h> 18#include <linux/dm-dirty-log.h> 19#include <linux/dm-kcopyd.h> 20#include <linux/dm-region-hash.h> 21 22#define DM_MSG_PREFIX "raid1" 23 24#define MAX_RECOVERY 1 /* Maximum number of regions recovered in parallel. */ 25 26#define DM_RAID1_HANDLE_ERRORS 0x01 27#define errors_handled(p) ((p)->features & DM_RAID1_HANDLE_ERRORS) 28 29static DECLARE_WAIT_QUEUE_HEAD(_kmirrord_recovery_stopped); 30 31/*----------------------------------------------------------------- 32 * Mirror set structures. 33 *---------------------------------------------------------------*/ 34enum dm_raid1_error { 35 DM_RAID1_WRITE_ERROR, 36 DM_RAID1_FLUSH_ERROR, 37 DM_RAID1_SYNC_ERROR, 38 DM_RAID1_READ_ERROR 39}; 40 41struct mirror { 42 struct mirror_set *ms; 43 atomic_t error_count; 44 unsigned long error_type; 45 struct dm_dev *dev; 46 sector_t offset; 47}; 48 49struct mirror_set { 50 struct dm_target *ti; 51 struct list_head list; 52 53 uint64_t features; 54 55 spinlock_t lock; /* protects the lists */ 56 struct bio_list reads; 57 struct bio_list writes; 58 struct bio_list failures; 59 struct bio_list holds; /* bios are waiting until suspend */ 60 61 struct dm_region_hash *rh; 62 struct dm_kcopyd_client *kcopyd_client; 63 struct dm_io_client *io_client; 64 65 /* recovery */ 66 region_t nr_regions; 67 int in_sync; 68 int log_failure; 69 int leg_failure; 70 atomic_t suspend; 71 72 atomic_t default_mirror; /* Default mirror */ 73 74 struct workqueue_struct *kmirrord_wq; 75 struct work_struct kmirrord_work; 76 struct timer_list timer; 77 unsigned long timer_pending; 78 79 struct work_struct trigger_event; 80 81 unsigned nr_mirrors; 82 struct mirror mirror[0]; 83}; 84 85DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(raid1_resync_throttle, 86 "A percentage of time allocated for raid resynchronization"); 87 88static void wakeup_mirrord(void *context) 89{ 90 struct mirror_set *ms = context; 91 92 queue_work(ms->kmirrord_wq, &ms->kmirrord_work); 93} 94 95static void delayed_wake_fn(unsigned long data) 96{ 97 struct mirror_set *ms = (struct mirror_set *) data; 98 99 clear_bit(0, &ms->timer_pending); 100 wakeup_mirrord(ms); 101} 102 103static void delayed_wake(struct mirror_set *ms) 104{ 105 if (test_and_set_bit(0, &ms->timer_pending)) 106 return; 107 108 ms->timer.expires = jiffies + HZ / 5; 109 ms->timer.data = (unsigned long) ms; 110 ms->timer.function = delayed_wake_fn; 111 add_timer(&ms->timer); 112} 113 114static void wakeup_all_recovery_waiters(void *context) 115{ 116 wake_up_all(&_kmirrord_recovery_stopped); 117} 118 119static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw) 120{ 121 unsigned long flags; 122 int should_wake = 0; 123 struct bio_list *bl; 124 125 bl = (rw == WRITE) ? &ms->writes : &ms->reads; 126 spin_lock_irqsave(&ms->lock, flags); 127 should_wake = !(bl->head); 128 bio_list_add(bl, bio); 129 spin_unlock_irqrestore(&ms->lock, flags); 130 131 if (should_wake) 132 wakeup_mirrord(ms); 133} 134 135static void dispatch_bios(void *context, struct bio_list *bio_list) 136{ 137 struct mirror_set *ms = context; 138 struct bio *bio; 139 140 while ((bio = bio_list_pop(bio_list))) 141 queue_bio(ms, bio, WRITE); 142} 143 144struct dm_raid1_bio_record { 145 struct mirror *m; 146 /* if details->bi_bdev == NULL, details were not saved */ 147 struct dm_bio_details details; 148 region_t write_region; 149}; 150 151/* 152 * Every mirror should look like this one. 153 */ 154#define DEFAULT_MIRROR 0 155 156/* 157 * This is yucky. We squirrel the mirror struct away inside 158 * bi_next for read/write buffers. This is safe since the bh 159 * doesn't get submitted to the lower levels of block layer. 160 */ 161static struct mirror *bio_get_m(struct bio *bio) 162{ 163 return (struct mirror *) bio->bi_next; 164} 165 166static void bio_set_m(struct bio *bio, struct mirror *m) 167{ 168 bio->bi_next = (struct bio *) m; 169} 170 171static struct mirror *get_default_mirror(struct mirror_set *ms) 172{ 173 return &ms->mirror[atomic_read(&ms->default_mirror)]; 174} 175 176static void set_default_mirror(struct mirror *m) 177{ 178 struct mirror_set *ms = m->ms; 179 struct mirror *m0 = &(ms->mirror[0]); 180 181 atomic_set(&ms->default_mirror, m - m0); 182} 183 184static struct mirror *get_valid_mirror(struct mirror_set *ms) 185{ 186 struct mirror *m; 187 188 for (m = ms->mirror; m < ms->mirror + ms->nr_mirrors; m++) 189 if (!atomic_read(&m->error_count)) 190 return m; 191 192 return NULL; 193} 194 195/* fail_mirror 196 * @m: mirror device to fail 197 * @error_type: one of the enum's, DM_RAID1_*_ERROR 198 * 199 * If errors are being handled, record the type of 200 * error encountered for this device. If this type 201 * of error has already been recorded, we can return; 202 * otherwise, we must signal userspace by triggering 203 * an event. Additionally, if the device is the 204 * primary device, we must choose a new primary, but 205 * only if the mirror is in-sync. 206 * 207 * This function must not block. 208 */ 209static void fail_mirror(struct mirror *m, enum dm_raid1_error error_type) 210{ 211 struct mirror_set *ms = m->ms; 212 struct mirror *new; 213 214 ms->leg_failure = 1; 215 216 /* 217 * error_count is used for nothing more than a 218 * simple way to tell if a device has encountered 219 * errors. 220 */ 221 atomic_inc(&m->error_count); 222 223 if (test_and_set_bit(error_type, &m->error_type)) 224 return; 225 226 if (!errors_handled(ms)) 227 return; 228 229 if (m != get_default_mirror(ms)) 230 goto out; 231 232 if (!ms->in_sync) { 233 /* 234 * Better to issue requests to same failing device 235 * than to risk returning corrupt data. 236 */ 237 DMERR("Primary mirror (%s) failed while out-of-sync: " 238 "Reads may fail.", m->dev->name); 239 goto out; 240 } 241 242 new = get_valid_mirror(ms); 243 if (new) 244 set_default_mirror(new); 245 else 246 DMWARN("All sides of mirror have failed."); 247 248out: 249 schedule_work(&ms->trigger_event); 250} 251 252static int mirror_flush(struct dm_target *ti) 253{ 254 struct mirror_set *ms = ti->private; 255 unsigned long error_bits; 256 257 unsigned int i; 258 struct dm_io_region io[ms->nr_mirrors]; 259 struct mirror *m; 260 struct dm_io_request io_req = { 261 .bi_rw = WRITE_FLUSH, 262 .mem.type = DM_IO_KMEM, 263 .mem.ptr.addr = NULL, 264 .client = ms->io_client, 265 }; 266 267 for (i = 0, m = ms->mirror; i < ms->nr_mirrors; i++, m++) { 268 io[i].bdev = m->dev->bdev; 269 io[i].sector = 0; 270 io[i].count = 0; 271 } 272 273 error_bits = -1; 274 dm_io(&io_req, ms->nr_mirrors, io, &error_bits); 275 if (unlikely(error_bits != 0)) { 276 for (i = 0; i < ms->nr_mirrors; i++) 277 if (test_bit(i, &error_bits)) 278 fail_mirror(ms->mirror + i, 279 DM_RAID1_FLUSH_ERROR); 280 return -EIO; 281 } 282 283 return 0; 284} 285 286/*----------------------------------------------------------------- 287 * Recovery. 288 * 289 * When a mirror is first activated we may find that some regions 290 * are in the no-sync state. We have to recover these by 291 * recopying from the default mirror to all the others. 292 *---------------------------------------------------------------*/ 293static void recovery_complete(int read_err, unsigned long write_err, 294 void *context) 295{ 296 struct dm_region *reg = context; 297 struct mirror_set *ms = dm_rh_region_context(reg); 298 int m, bit = 0; 299 300 if (read_err) { 301 /* Read error means the failure of default mirror. */ 302 DMERR_LIMIT("Unable to read primary mirror during recovery"); 303 fail_mirror(get_default_mirror(ms), DM_RAID1_SYNC_ERROR); 304 } 305 306 if (write_err) { 307 DMERR_LIMIT("Write error during recovery (error = 0x%lx)", 308 write_err); 309 /* 310 * Bits correspond to devices (excluding default mirror). 311 * The default mirror cannot change during recovery. 312 */ 313 for (m = 0; m < ms->nr_mirrors; m++) { 314 if (&ms->mirror[m] == get_default_mirror(ms)) 315 continue; 316 if (test_bit(bit, &write_err)) 317 fail_mirror(ms->mirror + m, 318 DM_RAID1_SYNC_ERROR); 319 bit++; 320 } 321 } 322 323 dm_rh_recovery_end(reg, !(read_err || write_err)); 324} 325 326static int recover(struct mirror_set *ms, struct dm_region *reg) 327{ 328 int r; 329 unsigned i; 330 struct dm_io_region from, to[DM_KCOPYD_MAX_REGIONS], *dest; 331 struct mirror *m; 332 unsigned long flags = 0; 333 region_t key = dm_rh_get_region_key(reg); 334 sector_t region_size = dm_rh_get_region_size(ms->rh); 335 336 /* fill in the source */ 337 m = get_default_mirror(ms); 338 from.bdev = m->dev->bdev; 339 from.sector = m->offset + dm_rh_region_to_sector(ms->rh, key); 340 if (key == (ms->nr_regions - 1)) { 341 /* 342 * The final region may be smaller than 343 * region_size. 344 */ 345 from.count = ms->ti->len & (region_size - 1); 346 if (!from.count) 347 from.count = region_size; 348 } else 349 from.count = region_size; 350 351 /* fill in the destinations */ 352 for (i = 0, dest = to; i < ms->nr_mirrors; i++) { 353 if (&ms->mirror[i] == get_default_mirror(ms)) 354 continue; 355 356 m = ms->mirror + i; 357 dest->bdev = m->dev->bdev; 358 dest->sector = m->offset + dm_rh_region_to_sector(ms->rh, key); 359 dest->count = from.count; 360 dest++; 361 } 362 363 /* hand to kcopyd */ 364 if (!errors_handled(ms)) 365 set_bit(DM_KCOPYD_IGNORE_ERROR, &flags); 366 367 r = dm_kcopyd_copy(ms->kcopyd_client, &from, ms->nr_mirrors - 1, to, 368 flags, recovery_complete, reg); 369 370 return r; 371} 372 373static void do_recovery(struct mirror_set *ms) 374{ 375 struct dm_region *reg; 376 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 377 int r; 378 379 /* 380 * Start quiescing some regions. 381 */ 382 dm_rh_recovery_prepare(ms->rh); 383 384 /* 385 * Copy any already quiesced regions. 386 */ 387 while ((reg = dm_rh_recovery_start(ms->rh))) { 388 r = recover(ms, reg); 389 if (r) 390 dm_rh_recovery_end(reg, 0); 391 } 392 393 /* 394 * Update the in sync flag. 395 */ 396 if (!ms->in_sync && 397 (log->type->get_sync_count(log) == ms->nr_regions)) { 398 /* the sync is complete */ 399 dm_table_event(ms->ti->table); 400 ms->in_sync = 1; 401 } 402} 403 404/*----------------------------------------------------------------- 405 * Reads 406 *---------------------------------------------------------------*/ 407static struct mirror *choose_mirror(struct mirror_set *ms, sector_t sector) 408{ 409 struct mirror *m = get_default_mirror(ms); 410 411 do { 412 if (likely(!atomic_read(&m->error_count))) 413 return m; 414 415 if (m-- == ms->mirror) 416 m += ms->nr_mirrors; 417 } while (m != get_default_mirror(ms)); 418 419 return NULL; 420} 421 422static int default_ok(struct mirror *m) 423{ 424 struct mirror *default_mirror = get_default_mirror(m->ms); 425 426 return !atomic_read(&default_mirror->error_count); 427} 428 429static int mirror_available(struct mirror_set *ms, struct bio *bio) 430{ 431 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 432 region_t region = dm_rh_bio_to_region(ms->rh, bio); 433 434 if (log->type->in_sync(log, region, 0)) 435 return choose_mirror(ms, bio->bi_iter.bi_sector) ? 1 : 0; 436 437 return 0; 438} 439 440/* 441 * remap a buffer to a particular mirror. 442 */ 443static sector_t map_sector(struct mirror *m, struct bio *bio) 444{ 445 if (unlikely(!bio->bi_iter.bi_size)) 446 return 0; 447 return m->offset + dm_target_offset(m->ms->ti, bio->bi_iter.bi_sector); 448} 449 450static void map_bio(struct mirror *m, struct bio *bio) 451{ 452 bio->bi_bdev = m->dev->bdev; 453 bio->bi_iter.bi_sector = map_sector(m, bio); 454} 455 456static void map_region(struct dm_io_region *io, struct mirror *m, 457 struct bio *bio) 458{ 459 io->bdev = m->dev->bdev; 460 io->sector = map_sector(m, bio); 461 io->count = bio_sectors(bio); 462} 463 464static void hold_bio(struct mirror_set *ms, struct bio *bio) 465{ 466 /* 467 * Lock is required to avoid race condition during suspend 468 * process. 469 */ 470 spin_lock_irq(&ms->lock); 471 472 if (atomic_read(&ms->suspend)) { 473 spin_unlock_irq(&ms->lock); 474 475 /* 476 * If device is suspended, complete the bio. 477 */ 478 if (dm_noflush_suspending(ms->ti)) 479 bio_endio(bio, DM_ENDIO_REQUEUE); 480 else 481 bio_endio(bio, -EIO); 482 return; 483 } 484 485 /* 486 * Hold bio until the suspend is complete. 487 */ 488 bio_list_add(&ms->holds, bio); 489 spin_unlock_irq(&ms->lock); 490} 491 492/*----------------------------------------------------------------- 493 * Reads 494 *---------------------------------------------------------------*/ 495static void read_callback(unsigned long error, void *context) 496{ 497 struct bio *bio = context; 498 struct mirror *m; 499 500 m = bio_get_m(bio); 501 bio_set_m(bio, NULL); 502 503 if (likely(!error)) { 504 bio_endio(bio, 0); 505 return; 506 } 507 508 fail_mirror(m, DM_RAID1_READ_ERROR); 509 510 if (likely(default_ok(m)) || mirror_available(m->ms, bio)) { 511 DMWARN_LIMIT("Read failure on mirror device %s. " 512 "Trying alternative device.", 513 m->dev->name); 514 queue_bio(m->ms, bio, bio_rw(bio)); 515 return; 516 } 517 518 DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.", 519 m->dev->name); 520 bio_endio(bio, -EIO); 521} 522 523/* Asynchronous read. */ 524static void read_async_bio(struct mirror *m, struct bio *bio) 525{ 526 struct dm_io_region io; 527 struct dm_io_request io_req = { 528 .bi_rw = READ, 529 .mem.type = DM_IO_BIO, 530 .mem.ptr.bio = bio, 531 .notify.fn = read_callback, 532 .notify.context = bio, 533 .client = m->ms->io_client, 534 }; 535 536 map_region(&io, m, bio); 537 bio_set_m(bio, m); 538 BUG_ON(dm_io(&io_req, 1, &io, NULL)); 539} 540 541static inline int region_in_sync(struct mirror_set *ms, region_t region, 542 int may_block) 543{ 544 int state = dm_rh_get_state(ms->rh, region, may_block); 545 return state == DM_RH_CLEAN || state == DM_RH_DIRTY; 546} 547 548static void do_reads(struct mirror_set *ms, struct bio_list *reads) 549{ 550 region_t region; 551 struct bio *bio; 552 struct mirror *m; 553 554 while ((bio = bio_list_pop(reads))) { 555 region = dm_rh_bio_to_region(ms->rh, bio); 556 m = get_default_mirror(ms); 557 558 /* 559 * We can only read balance if the region is in sync. 560 */ 561 if (likely(region_in_sync(ms, region, 1))) 562 m = choose_mirror(ms, bio->bi_iter.bi_sector); 563 else if (m && atomic_read(&m->error_count)) 564 m = NULL; 565 566 if (likely(m)) 567 read_async_bio(m, bio); 568 else 569 bio_endio(bio, -EIO); 570 } 571} 572 573/*----------------------------------------------------------------- 574 * Writes. 575 * 576 * We do different things with the write io depending on the 577 * state of the region that it's in: 578 * 579 * SYNC: increment pending, use kcopyd to write to *all* mirrors 580 * RECOVERING: delay the io until recovery completes 581 * NOSYNC: increment pending, just write to the default mirror 582 *---------------------------------------------------------------*/ 583 584 585static void write_callback(unsigned long error, void *context) 586{ 587 unsigned i, ret = 0; 588 struct bio *bio = (struct bio *) context; 589 struct mirror_set *ms; 590 int should_wake = 0; 591 unsigned long flags; 592 593 ms = bio_get_m(bio)->ms; 594 bio_set_m(bio, NULL); 595 596 /* 597 * NOTE: We don't decrement the pending count here, 598 * instead it is done by the targets endio function. 599 * This way we handle both writes to SYNC and NOSYNC 600 * regions with the same code. 601 */ 602 if (likely(!error)) { 603 bio_endio(bio, ret); 604 return; 605 } 606 607 /* 608 * If the bio is discard, return an error, but do not 609 * degrade the array. 610 */ 611 if (bio->bi_rw & REQ_DISCARD) { 612 bio_endio(bio, -EOPNOTSUPP); 613 return; 614 } 615 616 for (i = 0; i < ms->nr_mirrors; i++) 617 if (test_bit(i, &error)) 618 fail_mirror(ms->mirror + i, DM_RAID1_WRITE_ERROR); 619 620 /* 621 * Need to raise event. Since raising 622 * events can block, we need to do it in 623 * the main thread. 624 */ 625 spin_lock_irqsave(&ms->lock, flags); 626 if (!ms->failures.head) 627 should_wake = 1; 628 bio_list_add(&ms->failures, bio); 629 spin_unlock_irqrestore(&ms->lock, flags); 630 if (should_wake) 631 wakeup_mirrord(ms); 632} 633 634static void do_write(struct mirror_set *ms, struct bio *bio) 635{ 636 unsigned int i; 637 struct dm_io_region io[ms->nr_mirrors], *dest = io; 638 struct mirror *m; 639 struct dm_io_request io_req = { 640 .bi_rw = WRITE | (bio->bi_rw & WRITE_FLUSH_FUA), 641 .mem.type = DM_IO_BIO, 642 .mem.ptr.bio = bio, 643 .notify.fn = write_callback, 644 .notify.context = bio, 645 .client = ms->io_client, 646 }; 647 648 if (bio->bi_rw & REQ_DISCARD) { 649 io_req.bi_rw |= REQ_DISCARD; 650 io_req.mem.type = DM_IO_KMEM; 651 io_req.mem.ptr.addr = NULL; 652 } 653 654 for (i = 0, m = ms->mirror; i < ms->nr_mirrors; i++, m++) 655 map_region(dest++, m, bio); 656 657 /* 658 * Use default mirror because we only need it to retrieve the reference 659 * to the mirror set in write_callback(). 660 */ 661 bio_set_m(bio, get_default_mirror(ms)); 662 663 BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL)); 664} 665 666static void do_writes(struct mirror_set *ms, struct bio_list *writes) 667{ 668 int state; 669 struct bio *bio; 670 struct bio_list sync, nosync, recover, *this_list = NULL; 671 struct bio_list requeue; 672 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 673 region_t region; 674 675 if (!writes->head) 676 return; 677 678 /* 679 * Classify each write. 680 */ 681 bio_list_init(&sync); 682 bio_list_init(&nosync); 683 bio_list_init(&recover); 684 bio_list_init(&requeue); 685 686 while ((bio = bio_list_pop(writes))) { 687 if ((bio->bi_rw & REQ_FLUSH) || 688 (bio->bi_rw & REQ_DISCARD)) { 689 bio_list_add(&sync, bio); 690 continue; 691 } 692 693 region = dm_rh_bio_to_region(ms->rh, bio); 694 695 if (log->type->is_remote_recovering && 696 log->type->is_remote_recovering(log, region)) { 697 bio_list_add(&requeue, bio); 698 continue; 699 } 700 701 state = dm_rh_get_state(ms->rh, region, 1); 702 switch (state) { 703 case DM_RH_CLEAN: 704 case DM_RH_DIRTY: 705 this_list = &sync; 706 break; 707 708 case DM_RH_NOSYNC: 709 this_list = &nosync; 710 break; 711 712 case DM_RH_RECOVERING: 713 this_list = &recover; 714 break; 715 } 716 717 bio_list_add(this_list, bio); 718 } 719 720 /* 721 * Add bios that are delayed due to remote recovery 722 * back on to the write queue 723 */ 724 if (unlikely(requeue.head)) { 725 spin_lock_irq(&ms->lock); 726 bio_list_merge(&ms->writes, &requeue); 727 spin_unlock_irq(&ms->lock); 728 delayed_wake(ms); 729 } 730 731 /* 732 * Increment the pending counts for any regions that will 733 * be written to (writes to recover regions are going to 734 * be delayed). 735 */ 736 dm_rh_inc_pending(ms->rh, &sync); 737 dm_rh_inc_pending(ms->rh, &nosync); 738 739 /* 740 * If the flush fails on a previous call and succeeds here, 741 * we must not reset the log_failure variable. We need 742 * userspace interaction to do that. 743 */ 744 ms->log_failure = dm_rh_flush(ms->rh) ? 1 : ms->log_failure; 745 746 /* 747 * Dispatch io. 748 */ 749 if (unlikely(ms->log_failure) && errors_handled(ms)) { 750 spin_lock_irq(&ms->lock); 751 bio_list_merge(&ms->failures, &sync); 752 spin_unlock_irq(&ms->lock); 753 wakeup_mirrord(ms); 754 } else 755 while ((bio = bio_list_pop(&sync))) 756 do_write(ms, bio); 757 758 while ((bio = bio_list_pop(&recover))) 759 dm_rh_delay(ms->rh, bio); 760 761 while ((bio = bio_list_pop(&nosync))) { 762 if (unlikely(ms->leg_failure) && errors_handled(ms)) { 763 spin_lock_irq(&ms->lock); 764 bio_list_add(&ms->failures, bio); 765 spin_unlock_irq(&ms->lock); 766 wakeup_mirrord(ms); 767 } else { 768 map_bio(get_default_mirror(ms), bio); 769 generic_make_request(bio); 770 } 771 } 772} 773 774static void do_failures(struct mirror_set *ms, struct bio_list *failures) 775{ 776 struct bio *bio; 777 778 if (likely(!failures->head)) 779 return; 780 781 /* 782 * If the log has failed, unattempted writes are being 783 * put on the holds list. We can't issue those writes 784 * until a log has been marked, so we must store them. 785 * 786 * If a 'noflush' suspend is in progress, we can requeue 787 * the I/O's to the core. This give userspace a chance 788 * to reconfigure the mirror, at which point the core 789 * will reissue the writes. If the 'noflush' flag is 790 * not set, we have no choice but to return errors. 791 * 792 * Some writes on the failures list may have been 793 * submitted before the log failure and represent a 794 * failure to write to one of the devices. It is ok 795 * for us to treat them the same and requeue them 796 * as well. 797 */ 798 while ((bio = bio_list_pop(failures))) { 799 if (!ms->log_failure) { 800 ms->in_sync = 0; 801 dm_rh_mark_nosync(ms->rh, bio); 802 } 803 804 /* 805 * If all the legs are dead, fail the I/O. 806 * If we have been told to handle errors, hold the bio 807 * and wait for userspace to deal with the problem. 808 * Otherwise pretend that the I/O succeeded. (This would 809 * be wrong if the failed leg returned after reboot and 810 * got replicated back to the good legs.) 811 */ 812 if (!get_valid_mirror(ms)) 813 bio_endio(bio, -EIO); 814 else if (errors_handled(ms)) 815 hold_bio(ms, bio); 816 else 817 bio_endio(bio, 0); 818 } 819} 820 821static void trigger_event(struct work_struct *work) 822{ 823 struct mirror_set *ms = 824 container_of(work, struct mirror_set, trigger_event); 825 826 dm_table_event(ms->ti->table); 827} 828 829/*----------------------------------------------------------------- 830 * kmirrord 831 *---------------------------------------------------------------*/ 832static void do_mirror(struct work_struct *work) 833{ 834 struct mirror_set *ms = container_of(work, struct mirror_set, 835 kmirrord_work); 836 struct bio_list reads, writes, failures; 837 unsigned long flags; 838 839 spin_lock_irqsave(&ms->lock, flags); 840 reads = ms->reads; 841 writes = ms->writes; 842 failures = ms->failures; 843 bio_list_init(&ms->reads); 844 bio_list_init(&ms->writes); 845 bio_list_init(&ms->failures); 846 spin_unlock_irqrestore(&ms->lock, flags); 847 848 dm_rh_update_states(ms->rh, errors_handled(ms)); 849 do_recovery(ms); 850 do_reads(ms, &reads); 851 do_writes(ms, &writes); 852 do_failures(ms, &failures); 853} 854 855/*----------------------------------------------------------------- 856 * Target functions 857 *---------------------------------------------------------------*/ 858static struct mirror_set *alloc_context(unsigned int nr_mirrors, 859 uint32_t region_size, 860 struct dm_target *ti, 861 struct dm_dirty_log *dl) 862{ 863 size_t len; 864 struct mirror_set *ms = NULL; 865 866 len = sizeof(*ms) + (sizeof(ms->mirror[0]) * nr_mirrors); 867 868 ms = kzalloc(len, GFP_KERNEL); 869 if (!ms) { 870 ti->error = "Cannot allocate mirror context"; 871 return NULL; 872 } 873 874 spin_lock_init(&ms->lock); 875 bio_list_init(&ms->reads); 876 bio_list_init(&ms->writes); 877 bio_list_init(&ms->failures); 878 bio_list_init(&ms->holds); 879 880 ms->ti = ti; 881 ms->nr_mirrors = nr_mirrors; 882 ms->nr_regions = dm_sector_div_up(ti->len, region_size); 883 ms->in_sync = 0; 884 ms->log_failure = 0; 885 ms->leg_failure = 0; 886 atomic_set(&ms->suspend, 0); 887 atomic_set(&ms->default_mirror, DEFAULT_MIRROR); 888 889 ms->io_client = dm_io_client_create(); 890 if (IS_ERR(ms->io_client)) { 891 ti->error = "Error creating dm_io client"; 892 kfree(ms); 893 return NULL; 894 } 895 896 ms->rh = dm_region_hash_create(ms, dispatch_bios, wakeup_mirrord, 897 wakeup_all_recovery_waiters, 898 ms->ti->begin, MAX_RECOVERY, 899 dl, region_size, ms->nr_regions); 900 if (IS_ERR(ms->rh)) { 901 ti->error = "Error creating dirty region hash"; 902 dm_io_client_destroy(ms->io_client); 903 kfree(ms); 904 return NULL; 905 } 906 907 return ms; 908} 909 910static void free_context(struct mirror_set *ms, struct dm_target *ti, 911 unsigned int m) 912{ 913 while (m--) 914 dm_put_device(ti, ms->mirror[m].dev); 915 916 dm_io_client_destroy(ms->io_client); 917 dm_region_hash_destroy(ms->rh); 918 kfree(ms); 919} 920 921static int get_mirror(struct mirror_set *ms, struct dm_target *ti, 922 unsigned int mirror, char **argv) 923{ 924 unsigned long long offset; 925 char dummy; 926 927 if (sscanf(argv[1], "%llu%c", &offset, &dummy) != 1) { 928 ti->error = "Invalid offset"; 929 return -EINVAL; 930 } 931 932 if (dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), 933 &ms->mirror[mirror].dev)) { 934 ti->error = "Device lookup failure"; 935 return -ENXIO; 936 } 937 938 ms->mirror[mirror].ms = ms; 939 atomic_set(&(ms->mirror[mirror].error_count), 0); 940 ms->mirror[mirror].error_type = 0; 941 ms->mirror[mirror].offset = offset; 942 943 return 0; 944} 945 946/* 947 * Create dirty log: log_type #log_params <log_params> 948 */ 949static struct dm_dirty_log *create_dirty_log(struct dm_target *ti, 950 unsigned argc, char **argv, 951 unsigned *args_used) 952{ 953 unsigned param_count; 954 struct dm_dirty_log *dl; 955 char dummy; 956 957 if (argc < 2) { 958 ti->error = "Insufficient mirror log arguments"; 959 return NULL; 960 } 961 962 if (sscanf(argv[1], "%u%c", ¶m_count, &dummy) != 1) { 963 ti->error = "Invalid mirror log argument count"; 964 return NULL; 965 } 966 967 *args_used = 2 + param_count; 968 969 if (argc < *args_used) { 970 ti->error = "Insufficient mirror log arguments"; 971 return NULL; 972 } 973 974 dl = dm_dirty_log_create(argv[0], ti, mirror_flush, param_count, 975 argv + 2); 976 if (!dl) { 977 ti->error = "Error creating mirror dirty log"; 978 return NULL; 979 } 980 981 return dl; 982} 983 984static int parse_features(struct mirror_set *ms, unsigned argc, char **argv, 985 unsigned *args_used) 986{ 987 unsigned num_features; 988 struct dm_target *ti = ms->ti; 989 char dummy; 990 991 *args_used = 0; 992 993 if (!argc) 994 return 0; 995 996 if (sscanf(argv[0], "%u%c", &num_features, &dummy) != 1) { 997 ti->error = "Invalid number of features"; 998 return -EINVAL; 999 } 1000 1001 argc--; 1002 argv++; 1003 (*args_used)++; 1004 1005 if (num_features > argc) { 1006 ti->error = "Not enough arguments to support feature count"; 1007 return -EINVAL; 1008 } 1009 1010 if (!strcmp("handle_errors", argv[0])) 1011 ms->features |= DM_RAID1_HANDLE_ERRORS; 1012 else { 1013 ti->error = "Unrecognised feature requested"; 1014 return -EINVAL; 1015 } 1016 1017 (*args_used)++; 1018 1019 return 0; 1020} 1021 1022/* 1023 * Construct a mirror mapping: 1024 * 1025 * log_type #log_params <log_params> 1026 * #mirrors [mirror_path offset]{2,} 1027 * [#features <features>] 1028 * 1029 * log_type is "core" or "disk" 1030 * #log_params is between 1 and 3 1031 * 1032 * If present, features must be "handle_errors". 1033 */ 1034static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv) 1035{ 1036 int r; 1037 unsigned int nr_mirrors, m, args_used; 1038 struct mirror_set *ms; 1039 struct dm_dirty_log *dl; 1040 char dummy; 1041 1042 dl = create_dirty_log(ti, argc, argv, &args_used); 1043 if (!dl) 1044 return -EINVAL; 1045 1046 argv += args_used; 1047 argc -= args_used; 1048 1049 if (!argc || sscanf(argv[0], "%u%c", &nr_mirrors, &dummy) != 1 || 1050 nr_mirrors < 2 || nr_mirrors > DM_KCOPYD_MAX_REGIONS + 1) { 1051 ti->error = "Invalid number of mirrors"; 1052 dm_dirty_log_destroy(dl); 1053 return -EINVAL; 1054 } 1055 1056 argv++, argc--; 1057 1058 if (argc < nr_mirrors * 2) { 1059 ti->error = "Too few mirror arguments"; 1060 dm_dirty_log_destroy(dl); 1061 return -EINVAL; 1062 } 1063 1064 ms = alloc_context(nr_mirrors, dl->type->get_region_size(dl), ti, dl); 1065 if (!ms) { 1066 dm_dirty_log_destroy(dl); 1067 return -ENOMEM; 1068 } 1069 1070 /* Get the mirror parameter sets */ 1071 for (m = 0; m < nr_mirrors; m++) { 1072 r = get_mirror(ms, ti, m, argv); 1073 if (r) { 1074 free_context(ms, ti, m); 1075 return r; 1076 } 1077 argv += 2; 1078 argc -= 2; 1079 } 1080 1081 ti->private = ms; 1082 1083 r = dm_set_target_max_io_len(ti, dm_rh_get_region_size(ms->rh)); 1084 if (r) 1085 goto err_free_context; 1086 1087 ti->num_flush_bios = 1; 1088 ti->num_discard_bios = 1; 1089 ti->per_bio_data_size = sizeof(struct dm_raid1_bio_record); 1090 ti->discard_zeroes_data_unsupported = true; 1091 1092 ms->kmirrord_wq = alloc_workqueue("kmirrord", WQ_MEM_RECLAIM, 0); 1093 if (!ms->kmirrord_wq) { 1094 DMERR("couldn't start kmirrord"); 1095 r = -ENOMEM; 1096 goto err_free_context; 1097 } 1098 INIT_WORK(&ms->kmirrord_work, do_mirror); 1099 init_timer(&ms->timer); 1100 ms->timer_pending = 0; 1101 INIT_WORK(&ms->trigger_event, trigger_event); 1102 1103 r = parse_features(ms, argc, argv, &args_used); 1104 if (r) 1105 goto err_destroy_wq; 1106 1107 argv += args_used; 1108 argc -= args_used; 1109 1110 /* 1111 * Any read-balancing addition depends on the 1112 * DM_RAID1_HANDLE_ERRORS flag being present. 1113 * This is because the decision to balance depends 1114 * on the sync state of a region. If the above 1115 * flag is not present, we ignore errors; and 1116 * the sync state may be inaccurate. 1117 */ 1118 1119 if (argc) { 1120 ti->error = "Too many mirror arguments"; 1121 r = -EINVAL; 1122 goto err_destroy_wq; 1123 } 1124 1125 ms->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); 1126 if (IS_ERR(ms->kcopyd_client)) { 1127 r = PTR_ERR(ms->kcopyd_client); 1128 goto err_destroy_wq; 1129 } 1130 1131 wakeup_mirrord(ms); 1132 return 0; 1133 1134err_destroy_wq: 1135 destroy_workqueue(ms->kmirrord_wq); 1136err_free_context: 1137 free_context(ms, ti, ms->nr_mirrors); 1138 return r; 1139} 1140 1141static void mirror_dtr(struct dm_target *ti) 1142{ 1143 struct mirror_set *ms = (struct mirror_set *) ti->private; 1144 1145 del_timer_sync(&ms->timer); 1146 flush_workqueue(ms->kmirrord_wq); 1147 flush_work(&ms->trigger_event); 1148 dm_kcopyd_client_destroy(ms->kcopyd_client); 1149 destroy_workqueue(ms->kmirrord_wq); 1150 free_context(ms, ti, ms->nr_mirrors); 1151} 1152 1153/* 1154 * Mirror mapping function 1155 */ 1156static int mirror_map(struct dm_target *ti, struct bio *bio) 1157{ 1158 int r, rw = bio_rw(bio); 1159 struct mirror *m; 1160 struct mirror_set *ms = ti->private; 1161 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 1162 struct dm_raid1_bio_record *bio_record = 1163 dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record)); 1164 1165 bio_record->details.bi_bdev = NULL; 1166 1167 if (rw == WRITE) { 1168 /* Save region for mirror_end_io() handler */ 1169 bio_record->write_region = dm_rh_bio_to_region(ms->rh, bio); 1170 queue_bio(ms, bio, rw); 1171 return DM_MAPIO_SUBMITTED; 1172 } 1173 1174 r = log->type->in_sync(log, dm_rh_bio_to_region(ms->rh, bio), 0); 1175 if (r < 0 && r != -EWOULDBLOCK) 1176 return r; 1177 1178 /* 1179 * If region is not in-sync queue the bio. 1180 */ 1181 if (!r || (r == -EWOULDBLOCK)) { 1182 if (rw == READA) 1183 return -EWOULDBLOCK; 1184 1185 queue_bio(ms, bio, rw); 1186 return DM_MAPIO_SUBMITTED; 1187 } 1188 1189 /* 1190 * The region is in-sync and we can perform reads directly. 1191 * Store enough information so we can retry if it fails. 1192 */ 1193 m = choose_mirror(ms, bio->bi_iter.bi_sector); 1194 if (unlikely(!m)) 1195 return -EIO; 1196 1197 dm_bio_record(&bio_record->details, bio); 1198 bio_record->m = m; 1199 1200 map_bio(m, bio); 1201 1202 return DM_MAPIO_REMAPPED; 1203} 1204 1205static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) 1206{ 1207 int rw = bio_rw(bio); 1208 struct mirror_set *ms = (struct mirror_set *) ti->private; 1209 struct mirror *m = NULL; 1210 struct dm_bio_details *bd = NULL; 1211 struct dm_raid1_bio_record *bio_record = 1212 dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record)); 1213 1214 /* 1215 * We need to dec pending if this was a write. 1216 */ 1217 if (rw == WRITE) { 1218 if (!(bio->bi_rw & (REQ_FLUSH | REQ_DISCARD))) 1219 dm_rh_dec(ms->rh, bio_record->write_region); 1220 return error; 1221 } 1222 1223 if (error == -EOPNOTSUPP) 1224 goto out; 1225 1226 if ((error == -EWOULDBLOCK) && (bio->bi_rw & REQ_RAHEAD)) 1227 goto out; 1228 1229 if (unlikely(error)) { 1230 if (!bio_record->details.bi_bdev) { 1231 /* 1232 * There wasn't enough memory to record necessary 1233 * information for a retry or there was no other 1234 * mirror in-sync. 1235 */ 1236 DMERR_LIMIT("Mirror read failed."); 1237 return -EIO; 1238 } 1239 1240 m = bio_record->m; 1241 1242 DMERR("Mirror read failed from %s. Trying alternative device.", 1243 m->dev->name); 1244 1245 fail_mirror(m, DM_RAID1_READ_ERROR); 1246 1247 /* 1248 * A failed read is requeued for another attempt using an intact 1249 * mirror. 1250 */ 1251 if (default_ok(m) || mirror_available(ms, bio)) { 1252 bd = &bio_record->details; 1253 1254 dm_bio_restore(bd, bio); 1255 bio_record->details.bi_bdev = NULL; 1256 1257 atomic_inc(&bio->bi_remaining); 1258 1259 queue_bio(ms, bio, rw); 1260 return DM_ENDIO_INCOMPLETE; 1261 } 1262 DMERR("All replicated volumes dead, failing I/O"); 1263 } 1264 1265out: 1266 bio_record->details.bi_bdev = NULL; 1267 1268 return error; 1269} 1270 1271static void mirror_presuspend(struct dm_target *ti) 1272{ 1273 struct mirror_set *ms = (struct mirror_set *) ti->private; 1274 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 1275 1276 struct bio_list holds; 1277 struct bio *bio; 1278 1279 atomic_set(&ms->suspend, 1); 1280 1281 /* 1282 * Process bios in the hold list to start recovery waiting 1283 * for bios in the hold list. After the process, no bio has 1284 * a chance to be added in the hold list because ms->suspend 1285 * is set. 1286 */ 1287 spin_lock_irq(&ms->lock); 1288 holds = ms->holds; 1289 bio_list_init(&ms->holds); 1290 spin_unlock_irq(&ms->lock); 1291 1292 while ((bio = bio_list_pop(&holds))) 1293 hold_bio(ms, bio); 1294 1295 /* 1296 * We must finish up all the work that we've 1297 * generated (i.e. recovery work). 1298 */ 1299 dm_rh_stop_recovery(ms->rh); 1300 1301 wait_event(_kmirrord_recovery_stopped, 1302 !dm_rh_recovery_in_flight(ms->rh)); 1303 1304 if (log->type->presuspend && log->type->presuspend(log)) 1305 /* FIXME: need better error handling */ 1306 DMWARN("log presuspend failed"); 1307 1308 /* 1309 * Now that recovery is complete/stopped and the 1310 * delayed bios are queued, we need to wait for 1311 * the worker thread to complete. This way, 1312 * we know that all of our I/O has been pushed. 1313 */ 1314 flush_workqueue(ms->kmirrord_wq); 1315} 1316 1317static void mirror_postsuspend(struct dm_target *ti) 1318{ 1319 struct mirror_set *ms = ti->private; 1320 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 1321 1322 if (log->type->postsuspend && log->type->postsuspend(log)) 1323 /* FIXME: need better error handling */ 1324 DMWARN("log postsuspend failed"); 1325} 1326 1327static void mirror_resume(struct dm_target *ti) 1328{ 1329 struct mirror_set *ms = ti->private; 1330 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 1331 1332 atomic_set(&ms->suspend, 0); 1333 if (log->type->resume && log->type->resume(log)) 1334 /* FIXME: need better error handling */ 1335 DMWARN("log resume failed"); 1336 dm_rh_start_recovery(ms->rh); 1337} 1338 1339/* 1340 * device_status_char 1341 * @m: mirror device/leg we want the status of 1342 * 1343 * We return one character representing the most severe error 1344 * we have encountered. 1345 * A => Alive - No failures 1346 * D => Dead - A write failure occurred leaving mirror out-of-sync 1347 * S => Sync - A sychronization failure occurred, mirror out-of-sync 1348 * R => Read - A read failure occurred, mirror data unaffected 1349 * 1350 * Returns: <char> 1351 */ 1352static char device_status_char(struct mirror *m) 1353{ 1354 if (!atomic_read(&(m->error_count))) 1355 return 'A'; 1356 1357 return (test_bit(DM_RAID1_FLUSH_ERROR, &(m->error_type))) ? 'F' : 1358 (test_bit(DM_RAID1_WRITE_ERROR, &(m->error_type))) ? 'D' : 1359 (test_bit(DM_RAID1_SYNC_ERROR, &(m->error_type))) ? 'S' : 1360 (test_bit(DM_RAID1_READ_ERROR, &(m->error_type))) ? 'R' : 'U'; 1361} 1362 1363 1364static void mirror_status(struct dm_target *ti, status_type_t type, 1365 unsigned status_flags, char *result, unsigned maxlen) 1366{ 1367 unsigned int m, sz = 0; 1368 struct mirror_set *ms = (struct mirror_set *) ti->private; 1369 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); 1370 char buffer[ms->nr_mirrors + 1]; 1371 1372 switch (type) { 1373 case STATUSTYPE_INFO: 1374 DMEMIT("%d ", ms->nr_mirrors); 1375 for (m = 0; m < ms->nr_mirrors; m++) { 1376 DMEMIT("%s ", ms->mirror[m].dev->name); 1377 buffer[m] = device_status_char(&(ms->mirror[m])); 1378 } 1379 buffer[m] = '\0'; 1380 1381 DMEMIT("%llu/%llu 1 %s ", 1382 (unsigned long long)log->type->get_sync_count(log), 1383 (unsigned long long)ms->nr_regions, buffer); 1384 1385 sz += log->type->status(log, type, result+sz, maxlen-sz); 1386 1387 break; 1388 1389 case STATUSTYPE_TABLE: 1390 sz = log->type->status(log, type, result, maxlen); 1391 1392 DMEMIT("%d", ms->nr_mirrors); 1393 for (m = 0; m < ms->nr_mirrors; m++) 1394 DMEMIT(" %s %llu", ms->mirror[m].dev->name, 1395 (unsigned long long)ms->mirror[m].offset); 1396 1397 if (ms->features & DM_RAID1_HANDLE_ERRORS) 1398 DMEMIT(" 1 handle_errors"); 1399 } 1400} 1401 1402static int mirror_iterate_devices(struct dm_target *ti, 1403 iterate_devices_callout_fn fn, void *data) 1404{ 1405 struct mirror_set *ms = ti->private; 1406 int ret = 0; 1407 unsigned i; 1408 1409 for (i = 0; !ret && i < ms->nr_mirrors; i++) 1410 ret = fn(ti, ms->mirror[i].dev, 1411 ms->mirror[i].offset, ti->len, data); 1412 1413 return ret; 1414} 1415 1416static struct target_type mirror_target = { 1417 .name = "mirror", 1418 .version = {1, 13, 2}, 1419 .module = THIS_MODULE, 1420 .ctr = mirror_ctr, 1421 .dtr = mirror_dtr, 1422 .map = mirror_map, 1423 .end_io = mirror_end_io, 1424 .presuspend = mirror_presuspend, 1425 .postsuspend = mirror_postsuspend, 1426 .resume = mirror_resume, 1427 .status = mirror_status, 1428 .iterate_devices = mirror_iterate_devices, 1429}; 1430 1431static int __init dm_mirror_init(void) 1432{ 1433 int r; 1434 1435 r = dm_register_target(&mirror_target); 1436 if (r < 0) { 1437 DMERR("Failed to register mirror target"); 1438 goto bad_target; 1439 } 1440 1441 return 0; 1442 1443bad_target: 1444 return r; 1445} 1446 1447static void __exit dm_mirror_exit(void) 1448{ 1449 dm_unregister_target(&mirror_target); 1450} 1451 1452/* Module hooks */ 1453module_init(dm_mirror_init); 1454module_exit(dm_mirror_exit); 1455 1456MODULE_DESCRIPTION(DM_NAME " mirror target"); 1457MODULE_AUTHOR("Joe Thornber"); 1458MODULE_LICENSE("GPL"); 1459