Lines Matching refs:conf
87 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) in stripe_hash() argument
90 return &conf->stripe_hashtbl[hash]; in stripe_hash()
98 static inline void lock_device_hash_lock(struct r5conf *conf, int hash) in lock_device_hash_lock() argument
100 spin_lock_irq(conf->hash_locks + hash); in lock_device_hash_lock()
101 spin_lock(&conf->device_lock); in lock_device_hash_lock()
104 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) in unlock_device_hash_lock() argument
106 spin_unlock(&conf->device_lock); in unlock_device_hash_lock()
107 spin_unlock_irq(conf->hash_locks + hash); in unlock_device_hash_lock()
110 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) in lock_all_device_hash_locks_irq() argument
114 spin_lock(conf->hash_locks); in lock_all_device_hash_locks_irq()
116 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); in lock_all_device_hash_locks_irq()
117 spin_lock(&conf->device_lock); in lock_all_device_hash_locks_irq()
120 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) in unlock_all_device_hash_locks_irq() argument
123 spin_unlock(&conf->device_lock); in unlock_all_device_hash_locks_irq()
125 spin_unlock(conf->hash_locks + i - 1); in unlock_all_device_hash_locks_irq()
241 static void print_raid5_conf (struct r5conf *conf);
252 struct r5conf *conf = sh->raid_conf; in raid5_wakeup_stripe_thread() local
264 group = conf->worker_groups + cpu_to_group(cpu); in raid5_wakeup_stripe_thread()
270 if (conf->worker_cnt_per_group == 0) { in raid5_wakeup_stripe_thread()
271 md_wakeup_thread(conf->mddev->thread); in raid5_wakeup_stripe_thread()
275 group = conf->worker_groups + cpu_to_group(sh->cpu); in raid5_wakeup_stripe_thread()
283 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { in raid5_wakeup_stripe_thread()
293 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, in do_release_stripe() argument
297 BUG_ON(atomic_read(&conf->active_stripes)==0); in do_release_stripe()
301 list_add_tail(&sh->lru, &conf->delayed_list); in do_release_stripe()
303 sh->bm_seq - conf->seq_write > 0) in do_release_stripe()
304 list_add_tail(&sh->lru, &conf->bitmap_list); in do_release_stripe()
308 if (conf->worker_cnt_per_group == 0) { in do_release_stripe()
309 list_add_tail(&sh->lru, &conf->handle_list); in do_release_stripe()
315 md_wakeup_thread(conf->mddev->thread); in do_release_stripe()
319 if (atomic_dec_return(&conf->preread_active_stripes) in do_release_stripe()
321 md_wakeup_thread(conf->mddev->thread); in do_release_stripe()
322 atomic_dec(&conf->active_stripes); in do_release_stripe()
328 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, in __release_stripe() argument
332 do_release_stripe(conf, sh, temp_inactive_list); in __release_stripe()
342 static void release_inactive_stripe_list(struct r5conf *conf, in release_inactive_stripe_list() argument
363 spin_lock_irqsave(conf->hash_locks + hash, flags); in release_inactive_stripe_list()
364 if (list_empty(conf->inactive_list + hash) && in release_inactive_stripe_list()
366 atomic_dec(&conf->empty_inactive_list_nr); in release_inactive_stripe_list()
367 list_splice_tail_init(list, conf->inactive_list + hash); in release_inactive_stripe_list()
369 spin_unlock_irqrestore(conf->hash_locks + hash, flags); in release_inactive_stripe_list()
376 wake_up(&conf->wait_for_stripe); in release_inactive_stripe_list()
377 if (conf->retry_read_aligned) in release_inactive_stripe_list()
378 md_wakeup_thread(conf->mddev->thread); in release_inactive_stripe_list()
383 static int release_stripe_list(struct r5conf *conf, in release_stripe_list() argument
390 head = llist_del_all(&conf->released_stripes); in release_stripe_list()
406 __release_stripe(conf, sh, &temp_inactive_list[hash]); in release_stripe_list()
415 struct r5conf *conf = sh->raid_conf; in release_stripe() local
426 if (unlikely(!conf->mddev->thread) || in release_stripe()
429 wakeup = llist_add(&sh->release_list, &conf->released_stripes); in release_stripe()
431 md_wakeup_thread(conf->mddev->thread); in release_stripe()
436 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { in release_stripe()
439 do_release_stripe(conf, sh, &list); in release_stripe()
440 spin_unlock(&conf->device_lock); in release_stripe()
441 release_inactive_stripe_list(conf, &list, hash); in release_stripe()
454 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) in insert_hash() argument
456 struct hlist_head *hp = stripe_hash(conf, sh->sector); in insert_hash()
465 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) in get_free_stripe() argument
470 if (list_empty(conf->inactive_list + hash)) in get_free_stripe()
472 first = (conf->inactive_list + hash)->next; in get_free_stripe()
476 atomic_inc(&conf->active_stripes); in get_free_stripe()
478 if (list_empty(conf->inactive_list + hash)) in get_free_stripe()
479 atomic_inc(&conf->empty_inactive_list_nr); in get_free_stripe()
518 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
523 struct r5conf *conf = sh->raid_conf; in init_stripe() local
534 seq = read_seqcount_begin(&conf->gen_lock); in init_stripe()
535 sh->generation = conf->generation - previous; in init_stripe()
536 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; in init_stripe()
538 stripe_set_idx(sector, conf, previous, sh); in init_stripe()
555 if (read_seqcount_retry(&conf->gen_lock, seq)) in init_stripe()
558 insert_hash(conf, sh); in init_stripe()
563 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, in __find_stripe() argument
569 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) in __find_stripe()
589 static int calc_degraded(struct r5conf *conf) in calc_degraded() argument
596 for (i = 0; i < conf->previous_raid_disks; i++) { in calc_degraded()
597 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); in calc_degraded()
599 rdev = rcu_dereference(conf->disks[i].replacement); in calc_degraded()
614 if (conf->raid_disks >= conf->previous_raid_disks) in calc_degraded()
618 if (conf->raid_disks == conf->previous_raid_disks) in calc_degraded()
622 for (i = 0; i < conf->raid_disks; i++) { in calc_degraded()
623 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); in calc_degraded()
625 rdev = rcu_dereference(conf->disks[i].replacement); in calc_degraded()
636 if (conf->raid_disks <= conf->previous_raid_disks) in calc_degraded()
645 static int has_failed(struct r5conf *conf) in has_failed() argument
649 if (conf->mddev->reshape_position == MaxSector) in has_failed()
650 return conf->mddev->degraded > conf->max_degraded; in has_failed()
652 degraded = calc_degraded(conf); in has_failed()
653 if (degraded > conf->max_degraded) in has_failed()
659 get_active_stripe(struct r5conf *conf, sector_t sector, in get_active_stripe() argument
667 spin_lock_irq(conf->hash_locks + hash); in get_active_stripe()
670 wait_event_lock_irq(conf->wait_for_stripe, in get_active_stripe()
671 conf->quiesce == 0 || noquiesce, in get_active_stripe()
672 *(conf->hash_locks + hash)); in get_active_stripe()
673 sh = __find_stripe(conf, sector, conf->generation - previous); in get_active_stripe()
675 if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { in get_active_stripe()
676 sh = get_free_stripe(conf, hash); in get_active_stripe()
677 if (!sh && llist_empty(&conf->released_stripes) && in get_active_stripe()
678 !test_bit(R5_DID_ALLOC, &conf->cache_state)) in get_active_stripe()
680 &conf->cache_state); in get_active_stripe()
686 &conf->cache_state); in get_active_stripe()
688 conf->wait_for_stripe, in get_active_stripe()
689 !list_empty(conf->inactive_list + hash) && in get_active_stripe()
690 (atomic_read(&conf->active_stripes) in get_active_stripe()
691 < (conf->max_nr_stripes * 3 / 4) in get_active_stripe()
693 &conf->cache_state)), in get_active_stripe()
694 *(conf->hash_locks + hash)); in get_active_stripe()
696 &conf->cache_state); in get_active_stripe()
702 spin_lock(&conf->device_lock); in get_active_stripe()
705 atomic_inc(&conf->active_stripes); in get_active_stripe()
715 spin_unlock(&conf->device_lock); in get_active_stripe()
719 spin_unlock_irq(conf->hash_locks + hash); in get_active_stripe()
757 static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh) in stripe_add_to_batch_list() argument
768 if (!sector_div(tmp_sec, conf->chunk_sectors)) in stripe_add_to_batch_list()
773 spin_lock_irq(conf->hash_locks + hash); in stripe_add_to_batch_list()
774 head = __find_stripe(conf, head_sector, conf->generation); in stripe_add_to_batch_list()
776 spin_lock(&conf->device_lock); in stripe_add_to_batch_list()
779 atomic_inc(&conf->active_stripes); in stripe_add_to_batch_list()
789 spin_unlock(&conf->device_lock); in stripe_add_to_batch_list()
791 spin_unlock_irq(conf->hash_locks + hash); in stripe_add_to_batch_list()
837 if (atomic_dec_return(&conf->preread_active_stripes) in stripe_add_to_batch_list()
839 md_wakeup_thread(conf->mddev->thread); in stripe_add_to_batch_list()
860 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) in use_new_offset() argument
862 sector_t progress = conf->reshape_progress; in use_new_offset()
870 if (sh->generation == conf->generation - 1) in use_new_offset()
885 struct r5conf *conf = sh->raid_conf; in ops_run_io() local
921 rrdev = rcu_dereference(conf->disks[i].replacement); in ops_run_io()
923 rdev = rcu_dereference(conf->disks[i].rdev); in ops_run_io()
965 if (!conf->mddev->external && in ops_run_io()
966 conf->mddev->flags) { in ops_run_io()
971 md_check_recovery(conf->mddev); in ops_run_io()
979 md_wait_for_blocked_rdev(rdev, conf->mddev); in ops_run_io()
982 rdev_dec_pending(rdev, conf->mddev); in ops_run_io()
1008 if (use_new_offset(conf, sh)) in ops_run_io()
1033 if (conf->mddev->gendisk) in ops_run_io()
1035 bi, disk_devt(conf->mddev->gendisk), in ops_run_io()
1060 if (use_new_offset(conf, sh)) in ops_run_io()
1079 if (conf->mddev->gendisk) in ops_run_io()
1081 rbi, disk_devt(conf->mddev->gendisk), in ops_run_io()
1916 struct r5conf *conf = sh->raid_conf; in raid_run_ops() local
1917 int level = conf->level; in raid_run_ops()
1922 percpu = per_cpu_ptr(conf->percpu, cpu); in raid_run_ops()
1995 static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) in grow_one_stripe() argument
1999 sh = alloc_stripe(conf->slab_cache, gfp); in grow_one_stripe()
2003 sh->raid_conf = conf; in grow_one_stripe()
2007 kmem_cache_free(conf->slab_cache, sh); in grow_one_stripe()
2011 conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; in grow_one_stripe()
2013 atomic_inc(&conf->active_stripes); in grow_one_stripe()
2016 conf->max_nr_stripes++; in grow_one_stripe()
2020 static int grow_stripes(struct r5conf *conf, int num) in grow_stripes() argument
2023 int devs = max(conf->raid_disks, conf->previous_raid_disks); in grow_stripes()
2025 if (conf->mddev->gendisk) in grow_stripes()
2026 sprintf(conf->cache_name[0], in grow_stripes()
2027 "raid%d-%s", conf->level, mdname(conf->mddev)); in grow_stripes()
2029 sprintf(conf->cache_name[0], in grow_stripes()
2030 "raid%d-%p", conf->level, conf->mddev); in grow_stripes()
2031 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); in grow_stripes()
2033 conf->active_name = 0; in grow_stripes()
2034 sc = kmem_cache_create(conf->cache_name[conf->active_name], in grow_stripes()
2039 conf->slab_cache = sc; in grow_stripes()
2040 conf->pool_size = devs; in grow_stripes()
2042 if (!grow_one_stripe(conf, GFP_KERNEL)) in grow_stripes()
2078 static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) in resize_chunks() argument
2088 if (conf->scribble_disks >= new_disks && in resize_chunks()
2089 conf->scribble_sectors >= new_sectors) in resize_chunks()
2091 mddev_suspend(conf->mddev); in resize_chunks()
2097 percpu = per_cpu_ptr(conf->percpu, cpu); in resize_chunks()
2111 mddev_resume(conf->mddev); in resize_chunks()
2113 conf->scribble_disks = new_disks; in resize_chunks()
2114 conf->scribble_sectors = new_sectors; in resize_chunks()
2119 static int resize_stripes(struct r5conf *conf, int newsize) in resize_stripes() argument
2152 if (newsize <= conf->pool_size) in resize_stripes()
2155 err = md_allow_write(conf->mddev); in resize_stripes()
2160 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], in resize_stripes()
2167 mutex_lock(&conf->cache_size_mutex); in resize_stripes()
2169 for (i = conf->max_nr_stripes; i; i--) { in resize_stripes()
2174 nsh->raid_conf = conf; in resize_stripes()
2185 mutex_unlock(&conf->cache_size_mutex); in resize_stripes()
2195 lock_device_hash_lock(conf, hash); in resize_stripes()
2196 wait_event_cmd(conf->wait_for_stripe, in resize_stripes()
2197 !list_empty(conf->inactive_list + hash), in resize_stripes()
2198 unlock_device_hash_lock(conf, hash), in resize_stripes()
2199 lock_device_hash_lock(conf, hash)); in resize_stripes()
2200 osh = get_free_stripe(conf, hash); in resize_stripes()
2201 unlock_device_hash_lock(conf, hash); in resize_stripes()
2203 for(i=0; i<conf->pool_size; i++) { in resize_stripes()
2208 kmem_cache_free(conf->slab_cache, osh); in resize_stripes()
2210 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + in resize_stripes()
2211 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { in resize_stripes()
2216 kmem_cache_destroy(conf->slab_cache); in resize_stripes()
2225 for (i=0; i<conf->raid_disks; i++) in resize_stripes()
2226 ndisks[i] = conf->disks[i]; in resize_stripes()
2227 kfree(conf->disks); in resize_stripes()
2228 conf->disks = ndisks; in resize_stripes()
2232 mutex_unlock(&conf->cache_size_mutex); in resize_stripes()
2238 for (i=conf->raid_disks; i < newsize; i++) in resize_stripes()
2250 conf->slab_cache = sc; in resize_stripes()
2251 conf->active_name = 1-conf->active_name; in resize_stripes()
2253 conf->pool_size = newsize; in resize_stripes()
2257 static int drop_one_stripe(struct r5conf *conf) in drop_one_stripe() argument
2260 int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK; in drop_one_stripe()
2262 spin_lock_irq(conf->hash_locks + hash); in drop_one_stripe()
2263 sh = get_free_stripe(conf, hash); in drop_one_stripe()
2264 spin_unlock_irq(conf->hash_locks + hash); in drop_one_stripe()
2269 kmem_cache_free(conf->slab_cache, sh); in drop_one_stripe()
2270 atomic_dec(&conf->active_stripes); in drop_one_stripe()
2271 conf->max_nr_stripes--; in drop_one_stripe()
2275 static void shrink_stripes(struct r5conf *conf) in shrink_stripes() argument
2277 while (conf->max_nr_stripes && in shrink_stripes()
2278 drop_one_stripe(conf)) in shrink_stripes()
2281 if (conf->slab_cache) in shrink_stripes()
2282 kmem_cache_destroy(conf->slab_cache); in shrink_stripes()
2283 conf->slab_cache = NULL; in shrink_stripes()
2289 struct r5conf *conf = sh->raid_conf; in raid5_end_read_request() local
2313 rdev = conf->disks[i].replacement; in raid5_end_read_request()
2315 rdev = conf->disks[i].rdev; in raid5_end_read_request()
2317 if (use_new_offset(conf, sh)) in raid5_end_read_request()
2332 mdname(conf->mddev), STRIPE_SECTORS, in raid5_end_read_request()
2355 mdname(conf->mddev), in raid5_end_read_request()
2358 else if (conf->mddev->degraded >= conf->max_degraded) { in raid5_end_read_request()
2364 mdname(conf->mddev), in raid5_end_read_request()
2374 mdname(conf->mddev), in raid5_end_read_request()
2378 > conf->max_nr_stripes) in raid5_end_read_request()
2381 mdname(conf->mddev), bdn); in raid5_end_read_request()
2400 md_error(conf->mddev, rdev); in raid5_end_read_request()
2403 rdev_dec_pending(rdev, conf->mddev); in raid5_end_read_request()
2412 struct r5conf *conf = sh->raid_conf; in raid5_end_write_request() local
2422 rdev = conf->disks[i].rdev; in raid5_end_write_request()
2426 rdev = conf->disks[i].replacement; in raid5_end_write_request()
2434 rdev = conf->disks[i].rdev; in raid5_end_write_request()
2448 md_error(conf->mddev, rdev); in raid5_end_write_request()
2473 rdev_dec_pending(rdev, conf->mddev); in raid5_end_write_request()
2510 struct r5conf *conf = mddev->private; in error() local
2514 spin_lock_irqsave(&conf->device_lock, flags); in error()
2516 mddev->degraded = calc_degraded(conf); in error()
2517 spin_unlock_irqrestore(&conf->device_lock, flags); in error()
2529 conf->raid_disks - mddev->degraded); in error()
2536 static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, in raid5_compute_sector() argument
2546 int algorithm = previous ? conf->prev_algo in raid5_compute_sector()
2547 : conf->algorithm; in raid5_compute_sector()
2548 int sectors_per_chunk = previous ? conf->prev_chunk_sectors in raid5_compute_sector()
2549 : conf->chunk_sectors; in raid5_compute_sector()
2550 int raid_disks = previous ? conf->previous_raid_disks in raid5_compute_sector()
2551 : conf->raid_disks; in raid5_compute_sector()
2552 int data_disks = raid_disks - conf->max_degraded; in raid5_compute_sector()
2572 switch(conf->level) { in raid5_compute_sector()
2740 struct r5conf *conf = sh->raid_conf; in compute_blocknr() local
2742 int data_disks = raid_disks - conf->max_degraded; in compute_blocknr()
2744 int sectors_per_chunk = previous ? conf->prev_chunk_sectors in compute_blocknr()
2745 : conf->chunk_sectors; in compute_blocknr()
2746 int algorithm = previous ? conf->prev_algo in compute_blocknr()
2747 : conf->algorithm; in compute_blocknr()
2760 switch(conf->level) { in compute_blocknr()
2847 check = raid5_compute_sector(conf, r_sector, in compute_blocknr()
2852 mdname(conf->mddev)); in compute_blocknr()
2863 struct r5conf *conf = sh->raid_conf; in schedule_reconstruction() local
2864 int level = conf->level; in schedule_reconstruction()
2894 if (s->locked + conf->max_degraded == disks) in schedule_reconstruction()
2896 atomic_inc(&conf->pending_full_writes); in schedule_reconstruction()
2957 struct r5conf *conf = sh->raid_conf; in add_stripe_bio() local
3018 if (conf->mddev->bitmap && firstwrite) { in add_stripe_bio()
3033 bitmap_startwrite(conf->mddev->bitmap, sh->sector, in add_stripe_bio()
3038 sh->bm_seq = conf->seq_flush+1; in add_stripe_bio()
3045 stripe_add_to_batch_list(conf, sh); in add_stripe_bio()
3054 static void end_reshape(struct r5conf *conf);
3056 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, in stripe_set_idx() argument
3060 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; in stripe_set_idx()
3063 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; in stripe_set_idx()
3065 raid5_compute_sector(conf, in stripe_set_idx()
3066 stripe * (disks - conf->max_degraded) in stripe_set_idx()
3073 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, in handle_failed_stripe() argument
3086 rdev = rcu_dereference(conf->disks[i].rdev); in handle_failed_stripe()
3097 md_error(conf->mddev, rdev); in handle_failed_stripe()
3098 rdev_dec_pending(rdev, conf->mddev); in handle_failed_stripe()
3111 wake_up(&conf->wait_for_overlap); in handle_failed_stripe()
3118 md_write_end(conf->mddev); in handle_failed_stripe()
3125 bitmap_endwrite(conf->mddev->bitmap, sh->sector, in handle_failed_stripe()
3142 md_write_end(conf->mddev); in handle_failed_stripe()
3160 wake_up(&conf->wait_for_overlap); in handle_failed_stripe()
3174 bitmap_endwrite(conf->mddev->bitmap, sh->sector, in handle_failed_stripe()
3183 if (atomic_dec_and_test(&conf->pending_full_writes)) in handle_failed_stripe()
3184 md_wakeup_thread(conf->mddev->thread); in handle_failed_stripe()
3188 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, in handle_failed_sync() argument
3197 wake_up(&conf->wait_for_overlap); in handle_failed_sync()
3207 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { in handle_failed_sync()
3211 for (i = 0; i < conf->raid_disks; i++) { in handle_failed_sync()
3212 struct md_rdev *rdev = conf->disks[i].rdev; in handle_failed_sync()
3219 rdev = conf->disks[i].replacement; in handle_failed_sync()
3228 conf->recovery_disabled = in handle_failed_sync()
3229 conf->mddev->recovery_disabled; in handle_failed_sync()
3231 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); in handle_failed_sync()
3447 static void handle_stripe_clean_event(struct r5conf *conf, in handle_stripe_clean_event() argument
3481 md_write_end(conf->mddev); in handle_stripe_clean_event()
3487 bitmap_endwrite(conf->mddev->bitmap, sh->sector, in handle_stripe_clean_event()
3525 spin_lock_irq(conf->hash_locks + hash); in handle_stripe_clean_event()
3527 spin_unlock_irq(conf->hash_locks + hash); in handle_stripe_clean_event()
3542 if (atomic_dec_and_test(&conf->pending_full_writes)) in handle_stripe_clean_event()
3543 md_wakeup_thread(conf->mddev->thread); in handle_stripe_clean_event()
3549 static void handle_stripe_dirtying(struct r5conf *conf, in handle_stripe_dirtying() argument
3555 sector_t recovery_cp = conf->mddev->recovery_cp; in handle_stripe_dirtying()
3564 if (conf->rmw_level == PARITY_DISABLE_RMW || in handle_stripe_dirtying()
3572 conf->rmw_level, (unsigned long long)recovery_cp, in handle_stripe_dirtying()
3601 if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) { in handle_stripe_dirtying()
3603 if (conf->mddev->queue) in handle_stripe_dirtying()
3604 blk_add_trace_msg(conf->mddev->queue, in handle_stripe_dirtying()
3628 if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) { in handle_stripe_dirtying()
3655 if (rcw && conf->mddev->queue) in handle_stripe_dirtying()
3656 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", in handle_stripe_dirtying()
3681 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, in handle_parity_checks5() argument
3743 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); in handle_parity_checks5()
3744 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) in handle_parity_checks5()
3769 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, in handle_parity_checks6() argument
3895 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); in handle_parity_checks6()
3896 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) in handle_parity_checks6()
3933 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) in handle_stripe_expansion() argument
3950 sector_t s = raid5_compute_sector(conf, bn, 0, in handle_stripe_expansion()
3952 sh2 = get_active_stripe(conf, s, 0, 1, 1); in handle_stripe_expansion()
3974 for (j = 0; j < conf->raid_disks; j++) in handle_stripe_expansion()
3979 if (j == conf->raid_disks) { in handle_stripe_expansion()
4006 struct r5conf *conf = sh->raid_conf; in analyse_stripe() local
4065 rdev = rcu_dereference(conf->disks[i].replacement); in analyse_stripe()
4074 rdev = rcu_dereference(conf->disks[i].rdev); in analyse_stripe()
4122 conf->disks[i].rdev); in analyse_stripe()
4135 conf->disks[i].rdev); in analyse_stripe()
4144 conf->disks[i].replacement); in analyse_stripe()
4176 sh->sector >= conf->mddev->recovery_cp || in analyse_stripe()
4177 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) in analyse_stripe()
4285 struct r5conf *conf = sh->raid_conf; in handle_stripe() local
4340 rdev_dec_pending(s.blocked_rdev, conf->mddev); in handle_stripe()
4356 if (s.failed > conf->max_degraded) { in handle_stripe()
4361 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); in handle_stripe()
4363 handle_failed_sync(conf, sh, &s); in handle_stripe()
4415 || conf->level < 6; in handle_stripe()
4426 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); in handle_stripe()
4433 || (conf->level == 6 && s.to_write && s.failed) in handle_stripe()
4446 handle_stripe_dirtying(conf, sh, &s, disks); in handle_stripe()
4457 if (conf->level == 6) in handle_stripe()
4458 handle_parity_checks6(conf, sh, &s, disks); in handle_stripe()
4460 handle_parity_checks5(conf, sh, &s, disks); in handle_stripe()
4467 for (i = 0; i < conf->raid_disks; i++) in handle_stripe()
4481 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); in handle_stripe()
4484 wake_up(&conf->wait_for_overlap); in handle_stripe()
4490 if (s.failed <= conf->max_degraded && !conf->mddev->ro) in handle_stripe()
4514 = get_active_stripe(conf, sh->sector, 1, 1, 1); in handle_stripe()
4523 atomic_inc(&conf->preread_active_stripes); in handle_stripe()
4532 for (i = conf->raid_disks; i--; ) { in handle_stripe()
4542 sh->disks = conf->raid_disks; in handle_stripe()
4543 stripe_set_idx(sh->sector, conf, 0, sh); in handle_stripe()
4547 atomic_dec(&conf->reshape_stripes); in handle_stripe()
4548 wake_up(&conf->wait_for_overlap); in handle_stripe()
4549 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); in handle_stripe()
4554 handle_stripe_expansion(conf, sh); in handle_stripe()
4559 if (conf->mddev->external) in handle_stripe()
4561 conf->mddev); in handle_stripe()
4568 conf->mddev); in handle_stripe()
4577 rdev = conf->disks[i].rdev; in handle_stripe()
4580 md_error(conf->mddev, rdev); in handle_stripe()
4581 rdev_dec_pending(rdev, conf->mddev); in handle_stripe()
4584 rdev = conf->disks[i].rdev; in handle_stripe()
4587 rdev_dec_pending(rdev, conf->mddev); in handle_stripe()
4590 rdev = conf->disks[i].replacement; in handle_stripe()
4593 rdev = conf->disks[i].rdev; in handle_stripe()
4596 rdev_dec_pending(rdev, conf->mddev); in handle_stripe()
4610 atomic_dec(&conf->preread_active_stripes); in handle_stripe()
4611 if (atomic_read(&conf->preread_active_stripes) < in handle_stripe()
4613 md_wakeup_thread(conf->mddev->thread); in handle_stripe()
4621 static void raid5_activate_delayed(struct r5conf *conf) in raid5_activate_delayed() argument
4623 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { in raid5_activate_delayed()
4624 while (!list_empty(&conf->delayed_list)) { in raid5_activate_delayed()
4625 struct list_head *l = conf->delayed_list.next; in raid5_activate_delayed()
4631 atomic_inc(&conf->preread_active_stripes); in raid5_activate_delayed()
4632 list_add_tail(&sh->lru, &conf->hold_list); in raid5_activate_delayed()
4638 static void activate_bit_delay(struct r5conf *conf, in activate_bit_delay() argument
4643 list_add(&head, &conf->bitmap_list); in activate_bit_delay()
4644 list_del_init(&conf->bitmap_list); in activate_bit_delay()
4651 __release_stripe(conf, sh, &temp_inactive_list[hash]); in activate_bit_delay()
4657 struct r5conf *conf = mddev->private; in raid5_congested() local
4663 if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) in raid5_congested()
4665 if (conf->quiesce) in raid5_congested()
4667 if (atomic_read(&conf->empty_inactive_list_nr)) in raid5_congested()
4718 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) in add_bio_to_retry() argument
4722 spin_lock_irqsave(&conf->device_lock, flags); in add_bio_to_retry()
4724 bi->bi_next = conf->retry_read_aligned_list; in add_bio_to_retry()
4725 conf->retry_read_aligned_list = bi; in add_bio_to_retry()
4727 spin_unlock_irqrestore(&conf->device_lock, flags); in add_bio_to_retry()
4728 md_wakeup_thread(conf->mddev->thread); in add_bio_to_retry()
4731 static struct bio *remove_bio_from_retry(struct r5conf *conf) in remove_bio_from_retry() argument
4735 bi = conf->retry_read_aligned; in remove_bio_from_retry()
4737 conf->retry_read_aligned = NULL; in remove_bio_from_retry()
4740 bi = conf->retry_read_aligned_list; in remove_bio_from_retry()
4742 conf->retry_read_aligned_list = bi->bi_next; in remove_bio_from_retry()
4764 struct r5conf *conf; in raid5_align_endio() local
4773 conf = mddev->private; in raid5_align_endio()
4775 rdev_dec_pending(rdev, conf->mddev); in raid5_align_endio()
4781 if (atomic_dec_and_test(&conf->active_aligned_reads)) in raid5_align_endio()
4782 wake_up(&conf->wait_for_stripe); in raid5_align_endio()
4788 add_bio_to_retry(raid_bi, conf); in raid5_align_endio()
4812 struct r5conf *conf = mddev->private; in chunk_aligned_read() local
4838 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, in chunk_aligned_read()
4843 rdev = rcu_dereference(conf->disks[dd_idx].replacement); in chunk_aligned_read()
4846 rdev = rcu_dereference(conf->disks[dd_idx].rdev); in chunk_aligned_read()
4876 spin_lock_irq(&conf->device_lock); in chunk_aligned_read()
4877 wait_event_lock_irq(conf->wait_for_stripe, in chunk_aligned_read()
4878 conf->quiesce == 0, in chunk_aligned_read()
4879 conf->device_lock); in chunk_aligned_read()
4880 atomic_inc(&conf->active_aligned_reads); in chunk_aligned_read()
4881 spin_unlock_irq(&conf->device_lock); in chunk_aligned_read()
4906 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) in __get_priority_stripe() argument
4912 if (conf->worker_cnt_per_group == 0) { in __get_priority_stripe()
4913 handle_list = &conf->handle_list; in __get_priority_stripe()
4915 handle_list = &conf->worker_groups[group].handle_list; in __get_priority_stripe()
4916 wg = &conf->worker_groups[group]; in __get_priority_stripe()
4919 for (i = 0; i < conf->group_cnt; i++) { in __get_priority_stripe()
4920 handle_list = &conf->worker_groups[i].handle_list; in __get_priority_stripe()
4921 wg = &conf->worker_groups[i]; in __get_priority_stripe()
4930 list_empty(&conf->hold_list) ? "empty" : "busy", in __get_priority_stripe()
4931 atomic_read(&conf->pending_full_writes), conf->bypass_count); in __get_priority_stripe()
4936 if (list_empty(&conf->hold_list)) in __get_priority_stripe()
4937 conf->bypass_count = 0; in __get_priority_stripe()
4939 if (conf->hold_list.next == conf->last_hold) in __get_priority_stripe()
4940 conf->bypass_count++; in __get_priority_stripe()
4942 conf->last_hold = conf->hold_list.next; in __get_priority_stripe()
4943 conf->bypass_count -= conf->bypass_threshold; in __get_priority_stripe()
4944 if (conf->bypass_count < 0) in __get_priority_stripe()
4945 conf->bypass_count = 0; in __get_priority_stripe()
4948 } else if (!list_empty(&conf->hold_list) && in __get_priority_stripe()
4949 ((conf->bypass_threshold && in __get_priority_stripe()
4950 conf->bypass_count > conf->bypass_threshold) || in __get_priority_stripe()
4951 atomic_read(&conf->pending_full_writes) == 0)) { in __get_priority_stripe()
4953 list_for_each_entry(tmp, &conf->hold_list, lru) { in __get_priority_stripe()
4954 if (conf->worker_cnt_per_group == 0 || in __get_priority_stripe()
4964 conf->bypass_count -= conf->bypass_threshold; in __get_priority_stripe()
4965 if (conf->bypass_count < 0) in __get_priority_stripe()
4966 conf->bypass_count = 0; in __get_priority_stripe()
4995 struct r5conf *conf = mddev->private; in raid5_unplug() local
5000 spin_lock_irq(&conf->device_lock); in raid5_unplug()
5016 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); in raid5_unplug()
5019 spin_unlock_irq(&conf->device_lock); in raid5_unplug()
5021 release_inactive_stripe_list(conf, cb->temp_inactive_list, in raid5_unplug()
5058 struct r5conf *conf = mddev->private; in make_discard_request() local
5074 stripe_sectors = conf->chunk_sectors * in make_discard_request()
5075 (conf->raid_disks - conf->max_degraded); in make_discard_request()
5080 logical_sector *= conf->chunk_sectors; in make_discard_request()
5081 last_sector *= conf->chunk_sectors; in make_discard_request()
5088 sh = get_active_stripe(conf, logical_sector, 0, 0, 0); in make_discard_request()
5089 prepare_to_wait(&conf->wait_for_overlap, &w, in make_discard_request()
5099 for (d = 0; d < conf->raid_disks; d++) { in make_discard_request()
5111 finish_wait(&conf->wait_for_overlap, &w); in make_discard_request()
5113 for (d = 0; d < conf->raid_disks; d++) { in make_discard_request()
5122 if (conf->mddev->bitmap) { in make_discard_request()
5124 d < conf->raid_disks - conf->max_degraded; in make_discard_request()
5130 sh->bm_seq = conf->seq_flush + 1; in make_discard_request()
5137 atomic_inc(&conf->preread_active_stripes); in make_discard_request()
5150 struct r5conf *conf = mddev->private; in make_request() local
5187 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); in make_request()
5194 seq = read_seqcount_begin(&conf->gen_lock); in make_request()
5197 prepare_to_wait(&conf->wait_for_overlap, &w, in make_request()
5199 if (unlikely(conf->reshape_progress != MaxSector)) { in make_request()
5208 spin_lock_irq(&conf->device_lock); in make_request()
5210 ? logical_sector < conf->reshape_progress in make_request()
5211 : logical_sector >= conf->reshape_progress) { in make_request()
5215 ? logical_sector < conf->reshape_safe in make_request()
5216 : logical_sector >= conf->reshape_safe) { in make_request()
5217 spin_unlock_irq(&conf->device_lock); in make_request()
5223 spin_unlock_irq(&conf->device_lock); in make_request()
5226 new_sector = raid5_compute_sector(conf, logical_sector, in make_request()
5233 sh = get_active_stripe(conf, new_sector, previous, in make_request()
5246 spin_lock_irq(&conf->device_lock); in make_request()
5248 ? logical_sector >= conf->reshape_progress in make_request()
5249 : logical_sector < conf->reshape_progress) in make_request()
5252 spin_unlock_irq(&conf->device_lock); in make_request()
5260 if (read_seqcount_retry(&conf->gen_lock, seq)) { in make_request()
5277 prepare_to_wait(&conf->wait_for_overlap, in make_request()
5304 atomic_inc(&conf->preread_active_stripes); in make_request()
5312 finish_wait(&conf->wait_for_overlap, &w); in make_request()
5339 struct r5conf *conf = mddev->private; in reshape_request() local
5342 int raid_disks = conf->previous_raid_disks; in reshape_request()
5343 int data_disks = raid_disks - conf->max_degraded; in reshape_request()
5344 int new_data_disks = conf->raid_disks - conf->max_degraded; in reshape_request()
5355 conf->reshape_progress < raid5_size(mddev, 0, 0)) { in reshape_request()
5357 - conf->reshape_progress; in reshape_request()
5359 conf->reshape_progress > 0) in reshape_request()
5360 sector_nr = conf->reshape_progress; in reshape_request()
5385 writepos = conf->reshape_progress; in reshape_request()
5387 readpos = conf->reshape_progress; in reshape_request()
5389 safepos = conf->reshape_safe; in reshape_request()
5405 BUG_ON(conf->reshape_progress == 0); in reshape_request()
5436 if (conf->min_offset_diff < 0) { in reshape_request()
5437 safepos += -conf->min_offset_diff; in reshape_request()
5438 readpos += -conf->min_offset_diff; in reshape_request()
5440 writepos += conf->min_offset_diff; in reshape_request()
5445 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { in reshape_request()
5447 wait_event(conf->wait_for_overlap, in reshape_request()
5448 atomic_read(&conf->reshape_stripes)==0 in reshape_request()
5450 if (atomic_read(&conf->reshape_stripes) != 0) in reshape_request()
5452 mddev->reshape_position = conf->reshape_progress; in reshape_request()
5454 conf->reshape_checkpoint = jiffies; in reshape_request()
5461 spin_lock_irq(&conf->device_lock); in reshape_request()
5462 conf->reshape_safe = mddev->reshape_position; in reshape_request()
5463 spin_unlock_irq(&conf->device_lock); in reshape_request()
5464 wake_up(&conf->wait_for_overlap); in reshape_request()
5472 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); in reshape_request()
5474 atomic_inc(&conf->reshape_stripes); in reshape_request()
5482 if (conf->level == 6 && in reshape_request()
5500 spin_lock_irq(&conf->device_lock); in reshape_request()
5502 conf->reshape_progress -= reshape_sectors * new_data_disks; in reshape_request()
5504 conf->reshape_progress += reshape_sectors * new_data_disks; in reshape_request()
5505 spin_unlock_irq(&conf->device_lock); in reshape_request()
5512 raid5_compute_sector(conf, stripe_addr*(new_data_disks), in reshape_request()
5515 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) in reshape_request()
5521 sh = get_active_stripe(conf, first_sector, 1, 0, 1); in reshape_request()
5542 wait_event(conf->wait_for_overlap, in reshape_request()
5543 atomic_read(&conf->reshape_stripes) == 0 in reshape_request()
5545 if (atomic_read(&conf->reshape_stripes) != 0) in reshape_request()
5547 mddev->reshape_position = conf->reshape_progress; in reshape_request()
5549 conf->reshape_checkpoint = jiffies; in reshape_request()
5557 spin_lock_irq(&conf->device_lock); in reshape_request()
5558 conf->reshape_safe = mddev->reshape_position; in reshape_request()
5559 spin_unlock_irq(&conf->device_lock); in reshape_request()
5560 wake_up(&conf->wait_for_overlap); in reshape_request()
5569 struct r5conf *conf = mddev->private; in sync_request() local
5580 end_reshape(conf); in sync_request()
5588 conf->fullsync = 0; in sync_request()
5595 wait_event(conf->wait_for_overlap, conf->quiesce != 2); in sync_request()
5610 if (mddev->degraded >= conf->max_degraded && in sync_request()
5617 !conf->fullsync && in sync_request()
5628 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); in sync_request()
5630 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); in sync_request()
5641 for (i = 0; i < conf->raid_disks; i++) { in sync_request()
5642 struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); in sync_request()
5659 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) in retry_aligned_read() argument
5680 sector = raid5_compute_sector(conf, logical_sector, in retry_aligned_read()
5693 sh = get_active_stripe(conf, sector, 0, 1, 1); in retry_aligned_read()
5698 conf->retry_read_aligned = raid_bio; in retry_aligned_read()
5705 conf->retry_read_aligned = raid_bio; in retry_aligned_read()
5720 if (atomic_dec_and_test(&conf->active_aligned_reads)) in retry_aligned_read()
5721 wake_up(&conf->wait_for_stripe); in retry_aligned_read()
5725 static int handle_active_stripes(struct r5conf *conf, int group, in handle_active_stripes() argument
5734 (sh = __get_priority_stripe(conf, group)) != NULL) in handle_active_stripes()
5745 spin_unlock_irq(&conf->device_lock); in handle_active_stripes()
5747 release_inactive_stripe_list(conf, temp_inactive_list, in handle_active_stripes()
5751 spin_lock_irq(&conf->device_lock); in handle_active_stripes()
5760 spin_lock_irq(&conf->device_lock); in handle_active_stripes()
5763 __release_stripe(conf, batch[i], &temp_inactive_list[hash]); in handle_active_stripes()
5772 struct r5conf *conf = group->conf; in raid5_do_work() local
5773 int group_id = group - conf->worker_groups; in raid5_do_work()
5781 spin_lock_irq(&conf->device_lock); in raid5_do_work()
5785 released = release_stripe_list(conf, worker->temp_inactive_list); in raid5_do_work()
5787 batch_size = handle_active_stripes(conf, group_id, worker, in raid5_do_work()
5796 spin_unlock_irq(&conf->device_lock); in raid5_do_work()
5812 struct r5conf *conf = mddev->private; in raid5d() local
5822 spin_lock_irq(&conf->device_lock); in raid5d()
5827 released = release_stripe_list(conf, conf->temp_inactive_list); in raid5d()
5829 clear_bit(R5_DID_ALLOC, &conf->cache_state); in raid5d()
5832 !list_empty(&conf->bitmap_list)) { in raid5d()
5834 conf->seq_flush++; in raid5d()
5835 spin_unlock_irq(&conf->device_lock); in raid5d()
5837 spin_lock_irq(&conf->device_lock); in raid5d()
5838 conf->seq_write = conf->seq_flush; in raid5d()
5839 activate_bit_delay(conf, conf->temp_inactive_list); in raid5d()
5841 raid5_activate_delayed(conf); in raid5d()
5843 while ((bio = remove_bio_from_retry(conf))) { in raid5d()
5845 spin_unlock_irq(&conf->device_lock); in raid5d()
5846 ok = retry_aligned_read(conf, bio); in raid5d()
5847 spin_lock_irq(&conf->device_lock); in raid5d()
5853 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, in raid5d()
5854 conf->temp_inactive_list); in raid5d()
5860 spin_unlock_irq(&conf->device_lock); in raid5d()
5862 spin_lock_irq(&conf->device_lock); in raid5d()
5867 spin_unlock_irq(&conf->device_lock); in raid5d()
5868 if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) && in raid5d()
5869 mutex_trylock(&conf->cache_size_mutex)) { in raid5d()
5870 grow_one_stripe(conf, __GFP_NOWARN); in raid5d()
5874 set_bit(R5_DID_ALLOC, &conf->cache_state); in raid5d()
5875 mutex_unlock(&conf->cache_size_mutex); in raid5d()
5887 struct r5conf *conf; in raid5_show_stripe_cache_size() local
5890 conf = mddev->private; in raid5_show_stripe_cache_size()
5891 if (conf) in raid5_show_stripe_cache_size()
5892 ret = sprintf(page, "%d\n", conf->min_nr_stripes); in raid5_show_stripe_cache_size()
5900 struct r5conf *conf = mddev->private; in raid5_set_cache_size() local
5906 conf->min_nr_stripes = size; in raid5_set_cache_size()
5907 mutex_lock(&conf->cache_size_mutex); in raid5_set_cache_size()
5908 while (size < conf->max_nr_stripes && in raid5_set_cache_size()
5909 drop_one_stripe(conf)) in raid5_set_cache_size()
5911 mutex_unlock(&conf->cache_size_mutex); in raid5_set_cache_size()
5918 mutex_lock(&conf->cache_size_mutex); in raid5_set_cache_size()
5919 while (size > conf->max_nr_stripes) in raid5_set_cache_size()
5920 if (!grow_one_stripe(conf, GFP_KERNEL)) in raid5_set_cache_size()
5922 mutex_unlock(&conf->cache_size_mutex); in raid5_set_cache_size()
5931 struct r5conf *conf; in raid5_store_stripe_cache_size() local
5942 conf = mddev->private; in raid5_store_stripe_cache_size()
5943 if (!conf) in raid5_store_stripe_cache_size()
5960 struct r5conf *conf = mddev->private; in raid5_show_rmw_level() local
5961 if (conf) in raid5_show_rmw_level()
5962 return sprintf(page, "%d\n", conf->rmw_level); in raid5_show_rmw_level()
5970 struct r5conf *conf = mddev->private; in raid5_store_rmw_level() local
5973 if (!conf) in raid5_store_rmw_level()
5990 conf->rmw_level = new; in raid5_store_rmw_level()
6003 struct r5conf *conf; in raid5_show_preread_threshold() local
6006 conf = mddev->private; in raid5_show_preread_threshold()
6007 if (conf) in raid5_show_preread_threshold()
6008 ret = sprintf(page, "%d\n", conf->bypass_threshold); in raid5_show_preread_threshold()
6016 struct r5conf *conf; in raid5_store_preread_threshold() local
6028 conf = mddev->private; in raid5_store_preread_threshold()
6029 if (!conf) in raid5_store_preread_threshold()
6031 else if (new > conf->min_nr_stripes) in raid5_store_preread_threshold()
6034 conf->bypass_threshold = new; in raid5_store_preread_threshold()
6048 struct r5conf *conf; in raid5_show_skip_copy() local
6051 conf = mddev->private; in raid5_show_skip_copy()
6052 if (conf) in raid5_show_skip_copy()
6053 ret = sprintf(page, "%d\n", conf->skip_copy); in raid5_show_skip_copy()
6061 struct r5conf *conf; in raid5_store_skip_copy() local
6074 conf = mddev->private; in raid5_store_skip_copy()
6075 if (!conf) in raid5_store_skip_copy()
6077 else if (new != conf->skip_copy) { in raid5_store_skip_copy()
6079 conf->skip_copy = new; in raid5_store_skip_copy()
6100 struct r5conf *conf = mddev->private; in stripe_cache_active_show() local
6101 if (conf) in stripe_cache_active_show()
6102 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); in stripe_cache_active_show()
6113 struct r5conf *conf; in raid5_show_group_thread_cnt() local
6116 conf = mddev->private; in raid5_show_group_thread_cnt()
6117 if (conf) in raid5_show_group_thread_cnt()
6118 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); in raid5_show_group_thread_cnt()
6123 static int alloc_thread_groups(struct r5conf *conf, int cnt,
6130 struct r5conf *conf; in raid5_store_group_thread_cnt() local
6144 conf = mddev->private; in raid5_store_group_thread_cnt()
6145 if (!conf) in raid5_store_group_thread_cnt()
6147 else if (new != conf->worker_cnt_per_group) { in raid5_store_group_thread_cnt()
6150 old_groups = conf->worker_groups; in raid5_store_group_thread_cnt()
6154 err = alloc_thread_groups(conf, new, in raid5_store_group_thread_cnt()
6158 spin_lock_irq(&conf->device_lock); in raid5_store_group_thread_cnt()
6159 conf->group_cnt = group_cnt; in raid5_store_group_thread_cnt()
6160 conf->worker_cnt_per_group = worker_cnt_per_group; in raid5_store_group_thread_cnt()
6161 conf->worker_groups = new_groups; in raid5_store_group_thread_cnt()
6162 spin_unlock_irq(&conf->device_lock); in raid5_store_group_thread_cnt()
6194 static int alloc_thread_groups(struct r5conf *conf, int cnt, in alloc_thread_groups() argument
6225 group->conf = conf; in alloc_thread_groups()
6241 static void free_thread_groups(struct r5conf *conf) in free_thread_groups() argument
6243 if (conf->worker_groups) in free_thread_groups()
6244 kfree(conf->worker_groups[0].workers); in free_thread_groups()
6245 kfree(conf->worker_groups); in free_thread_groups()
6246 conf->worker_groups = NULL; in free_thread_groups()
6252 struct r5conf *conf = mddev->private; in raid5_size() local
6258 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); in raid5_size()
6262 return sectors * (raid_disks - conf->max_degraded); in raid5_size()
6265 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) in free_scratch_buffer() argument
6274 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) in alloc_scratch_buffer() argument
6276 if (conf->level == 6 && !percpu->spare_page) in alloc_scratch_buffer()
6279 percpu->scribble = scribble_alloc(max(conf->raid_disks, in alloc_scratch_buffer()
6280 conf->previous_raid_disks), in alloc_scratch_buffer()
6281 max(conf->chunk_sectors, in alloc_scratch_buffer()
6282 conf->prev_chunk_sectors) in alloc_scratch_buffer()
6286 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { in alloc_scratch_buffer()
6287 free_scratch_buffer(conf, percpu); in alloc_scratch_buffer()
6294 static void raid5_free_percpu(struct r5conf *conf) in raid5_free_percpu() argument
6298 if (!conf->percpu) in raid5_free_percpu()
6302 unregister_cpu_notifier(&conf->cpu_notify); in raid5_free_percpu()
6307 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); in raid5_free_percpu()
6310 free_percpu(conf->percpu); in raid5_free_percpu()
6313 static void free_conf(struct r5conf *conf) in free_conf() argument
6315 if (conf->shrinker.seeks) in free_conf()
6316 unregister_shrinker(&conf->shrinker); in free_conf()
6317 free_thread_groups(conf); in free_conf()
6318 shrink_stripes(conf); in free_conf()
6319 raid5_free_percpu(conf); in free_conf()
6320 kfree(conf->disks); in free_conf()
6321 kfree(conf->stripe_hashtbl); in free_conf()
6322 kfree(conf); in free_conf()
6329 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); in raid456_cpu_notify() local
6331 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); in raid456_cpu_notify()
6336 if (alloc_scratch_buffer(conf, percpu)) { in raid456_cpu_notify()
6344 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); in raid456_cpu_notify()
6353 static int raid5_alloc_percpu(struct r5conf *conf) in raid5_alloc_percpu() argument
6358 conf->percpu = alloc_percpu(struct raid5_percpu); in raid5_alloc_percpu()
6359 if (!conf->percpu) in raid5_alloc_percpu()
6363 conf->cpu_notify.notifier_call = raid456_cpu_notify; in raid5_alloc_percpu()
6364 conf->cpu_notify.priority = 0; in raid5_alloc_percpu()
6365 err = register_cpu_notifier(&conf->cpu_notify); in raid5_alloc_percpu()
6372 err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); in raid5_alloc_percpu()
6382 conf->scribble_disks = max(conf->raid_disks, in raid5_alloc_percpu()
6383 conf->previous_raid_disks); in raid5_alloc_percpu()
6384 conf->scribble_sectors = max(conf->chunk_sectors, in raid5_alloc_percpu()
6385 conf->prev_chunk_sectors); in raid5_alloc_percpu()
6393 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); in raid5_cache_scan() local
6396 if (mutex_trylock(&conf->cache_size_mutex)) { in raid5_cache_scan()
6399 conf->max_nr_stripes > conf->min_nr_stripes) { in raid5_cache_scan()
6400 if (drop_one_stripe(conf) == 0) { in raid5_cache_scan()
6406 mutex_unlock(&conf->cache_size_mutex); in raid5_cache_scan()
6414 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); in raid5_cache_count() local
6416 if (conf->max_nr_stripes < conf->min_nr_stripes) in raid5_cache_count()
6419 return conf->max_nr_stripes - conf->min_nr_stripes; in raid5_cache_count()
6424 struct r5conf *conf; in setup_conf() local
6462 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); in setup_conf()
6463 if (conf == NULL) in setup_conf()
6466 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, in setup_conf()
6468 conf->group_cnt = group_cnt; in setup_conf()
6469 conf->worker_cnt_per_group = worker_cnt_per_group; in setup_conf()
6470 conf->worker_groups = new_group; in setup_conf()
6473 spin_lock_init(&conf->device_lock); in setup_conf()
6474 seqcount_init(&conf->gen_lock); in setup_conf()
6475 mutex_init(&conf->cache_size_mutex); in setup_conf()
6476 init_waitqueue_head(&conf->wait_for_stripe); in setup_conf()
6477 init_waitqueue_head(&conf->wait_for_overlap); in setup_conf()
6478 INIT_LIST_HEAD(&conf->handle_list); in setup_conf()
6479 INIT_LIST_HEAD(&conf->hold_list); in setup_conf()
6480 INIT_LIST_HEAD(&conf->delayed_list); in setup_conf()
6481 INIT_LIST_HEAD(&conf->bitmap_list); in setup_conf()
6482 init_llist_head(&conf->released_stripes); in setup_conf()
6483 atomic_set(&conf->active_stripes, 0); in setup_conf()
6484 atomic_set(&conf->preread_active_stripes, 0); in setup_conf()
6485 atomic_set(&conf->active_aligned_reads, 0); in setup_conf()
6486 conf->bypass_threshold = BYPASS_THRESHOLD; in setup_conf()
6487 conf->recovery_disabled = mddev->recovery_disabled - 1; in setup_conf()
6489 conf->raid_disks = mddev->raid_disks; in setup_conf()
6491 conf->previous_raid_disks = mddev->raid_disks; in setup_conf()
6493 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; in setup_conf()
6494 max_disks = max(conf->raid_disks, conf->previous_raid_disks); in setup_conf()
6496 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), in setup_conf()
6498 if (!conf->disks) in setup_conf()
6501 conf->mddev = mddev; in setup_conf()
6503 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) in setup_conf()
6511 spin_lock_init(conf->hash_locks); in setup_conf()
6513 spin_lock_init(conf->hash_locks + i); in setup_conf()
6516 INIT_LIST_HEAD(conf->inactive_list + i); in setup_conf()
6519 INIT_LIST_HEAD(conf->temp_inactive_list + i); in setup_conf()
6521 conf->level = mddev->new_level; in setup_conf()
6522 conf->chunk_sectors = mddev->new_chunk_sectors; in setup_conf()
6523 if (raid5_alloc_percpu(conf) != 0) in setup_conf()
6533 disk = conf->disks + raid_disk; in setup_conf()
6552 conf->fullsync = 1; in setup_conf()
6555 conf->level = mddev->new_level; in setup_conf()
6556 if (conf->level == 6) { in setup_conf()
6557 conf->max_degraded = 2; in setup_conf()
6559 conf->rmw_level = PARITY_ENABLE_RMW; in setup_conf()
6561 conf->rmw_level = PARITY_DISABLE_RMW; in setup_conf()
6563 conf->max_degraded = 1; in setup_conf()
6564 conf->rmw_level = PARITY_ENABLE_RMW; in setup_conf()
6566 conf->algorithm = mddev->new_layout; in setup_conf()
6567 conf->reshape_progress = mddev->reshape_position; in setup_conf()
6568 if (conf->reshape_progress != MaxSector) { in setup_conf()
6569 conf->prev_chunk_sectors = mddev->chunk_sectors; in setup_conf()
6570 conf->prev_algo = mddev->layout; in setup_conf()
6573 conf->min_nr_stripes = NR_STRIPES; in setup_conf()
6574 memory = conf->min_nr_stripes * (sizeof(struct stripe_head) + in setup_conf()
6576 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); in setup_conf()
6577 if (grow_stripes(conf, conf->min_nr_stripes)) { in setup_conf()
6590 conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4; in setup_conf()
6591 conf->shrinker.scan_objects = raid5_cache_scan; in setup_conf()
6592 conf->shrinker.count_objects = raid5_cache_count; in setup_conf()
6593 conf->shrinker.batch = 128; in setup_conf()
6594 conf->shrinker.flags = 0; in setup_conf()
6595 register_shrinker(&conf->shrinker); in setup_conf()
6598 conf->thread = md_register_thread(raid5d, mddev, pers_name); in setup_conf()
6599 if (!conf->thread) { in setup_conf()
6606 return conf; in setup_conf()
6609 if (conf) { in setup_conf()
6610 free_conf(conf); in setup_conf()
6644 struct r5conf *conf; in run() local
6762 conf = setup_conf(mddev); in run()
6764 conf = mddev->private; in run()
6766 if (IS_ERR(conf)) in run()
6767 return PTR_ERR(conf); in run()
6769 conf->min_offset_diff = min_offset_diff; in run()
6770 mddev->thread = conf->thread; in run()
6771 conf->thread = NULL; in run()
6772 mddev->private = conf; in run()
6774 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; in run()
6776 rdev = conf->disks[i].rdev; in run()
6777 if (!rdev && conf->disks[i].replacement) { in run()
6779 rdev = conf->disks[i].replacement; in run()
6780 conf->disks[i].replacement = NULL; in run()
6782 conf->disks[i].rdev = rdev; in run()
6786 if (conf->disks[i].replacement && in run()
6787 conf->reshape_progress != MaxSector) { in run()
6813 conf->algorithm, in run()
6814 conf->raid_disks, in run()
6815 conf->max_degraded)) in run()
6819 conf->prev_algo, in run()
6820 conf->previous_raid_disks, in run()
6821 conf->max_degraded)) in run()
6829 mddev->degraded = calc_degraded(conf); in run()
6831 if (has_failed(conf)) { in run()
6834 mdname(mddev), mddev->degraded, conf->raid_disks); in run()
6859 " devices, algorithm %d\n", mdname(mddev), conf->level, in run()
6865 mdname(mddev), conf->level, in run()
6869 print_raid5_conf(conf); in run()
6871 if (conf->reshape_progress != MaxSector) { in run()
6872 conf->reshape_safe = conf->reshape_progress; in run()
6873 atomic_set(&conf->reshape_stripes, 0); in run()
6899 int data_disks = conf->previous_raid_disks - conf->max_degraded; in run()
6908 (conf->raid_disks - conf->max_degraded)); in run()
6975 print_raid5_conf(conf); in run()
6976 free_conf(conf); in run()
6984 struct r5conf *conf = priv; in raid5_free() local
6986 free_conf(conf); in raid5_free()
6992 struct r5conf *conf = mddev->private; in status() local
6997 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); in status()
6998 for (i = 0; i < conf->raid_disks; i++) in status()
7000 conf->disks[i].rdev && in status()
7001 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); in status()
7005 static void print_raid5_conf (struct r5conf *conf) in print_raid5_conf() argument
7011 if (!conf) { in print_raid5_conf()
7015 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, in print_raid5_conf()
7016 conf->raid_disks, in print_raid5_conf()
7017 conf->raid_disks - conf->mddev->degraded); in print_raid5_conf()
7019 for (i = 0; i < conf->raid_disks; i++) { in print_raid5_conf()
7021 tmp = conf->disks + i; in print_raid5_conf()
7032 struct r5conf *conf = mddev->private; in raid5_spare_active() local
7037 for (i = 0; i < conf->raid_disks; i++) { in raid5_spare_active()
7038 tmp = conf->disks + i; in raid5_spare_active()
7065 spin_lock_irqsave(&conf->device_lock, flags); in raid5_spare_active()
7066 mddev->degraded = calc_degraded(conf); in raid5_spare_active()
7067 spin_unlock_irqrestore(&conf->device_lock, flags); in raid5_spare_active()
7068 print_raid5_conf(conf); in raid5_spare_active()
7074 struct r5conf *conf = mddev->private; in raid5_remove_disk() local
7078 struct disk_info *p = conf->disks + number; in raid5_remove_disk()
7080 print_raid5_conf(conf); in raid5_remove_disk()
7088 if (number >= conf->raid_disks && in raid5_remove_disk()
7089 conf->reshape_progress == MaxSector) in raid5_remove_disk()
7101 mddev->recovery_disabled != conf->recovery_disabled && in raid5_remove_disk()
7102 !has_failed(conf) && in raid5_remove_disk()
7104 number < conf->raid_disks) { in raid5_remove_disk()
7130 print_raid5_conf(conf); in raid5_remove_disk()
7136 struct r5conf *conf = mddev->private; in raid5_add_disk() local
7141 int last = conf->raid_disks - 1; in raid5_add_disk()
7143 if (mddev->recovery_disabled == conf->recovery_disabled) in raid5_add_disk()
7146 if (rdev->saved_raid_disk < 0 && has_failed(conf)) in raid5_add_disk()
7159 conf->disks[rdev->saved_raid_disk].rdev == NULL) in raid5_add_disk()
7163 p = conf->disks + disk; in raid5_add_disk()
7169 conf->fullsync = 1; in raid5_add_disk()
7175 p = conf->disks + disk; in raid5_add_disk()
7182 conf->fullsync = 1; in raid5_add_disk()
7188 print_raid5_conf(conf); in raid5_add_disk()
7235 struct r5conf *conf = mddev->private; in check_stripe_cache() local
7237 > conf->min_nr_stripes || in check_stripe_cache()
7239 > conf->min_nr_stripes) { in check_stripe_cache()
7251 struct r5conf *conf = mddev->private; in check_reshape() local
7257 if (has_failed(conf)) in check_reshape()
7277 if (resize_chunks(conf, in check_reshape()
7278 conf->previous_raid_disks in check_reshape()
7284 return resize_stripes(conf, (conf->previous_raid_disks in check_reshape()
7290 struct r5conf *conf = mddev->private; in raid5_start_reshape() local
7301 if (has_failed(conf)) in raid5_start_reshape()
7310 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) in raid5_start_reshape()
7320 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) in raid5_start_reshape()
7327 atomic_set(&conf->reshape_stripes, 0); in raid5_start_reshape()
7328 spin_lock_irq(&conf->device_lock); in raid5_start_reshape()
7329 write_seqcount_begin(&conf->gen_lock); in raid5_start_reshape()
7330 conf->previous_raid_disks = conf->raid_disks; in raid5_start_reshape()
7331 conf->raid_disks += mddev->delta_disks; in raid5_start_reshape()
7332 conf->prev_chunk_sectors = conf->chunk_sectors; in raid5_start_reshape()
7333 conf->chunk_sectors = mddev->new_chunk_sectors; in raid5_start_reshape()
7334 conf->prev_algo = conf->algorithm; in raid5_start_reshape()
7335 conf->algorithm = mddev->new_layout; in raid5_start_reshape()
7336 conf->generation++; in raid5_start_reshape()
7342 conf->reshape_progress = raid5_size(mddev, 0, 0); in raid5_start_reshape()
7344 conf->reshape_progress = 0; in raid5_start_reshape()
7345 conf->reshape_safe = conf->reshape_progress; in raid5_start_reshape()
7346 write_seqcount_end(&conf->gen_lock); in raid5_start_reshape()
7347 spin_unlock_irq(&conf->device_lock); in raid5_start_reshape()
7369 >= conf->previous_raid_disks) in raid5_start_reshape()
7377 } else if (rdev->raid_disk >= conf->previous_raid_disks in raid5_start_reshape()
7387 spin_lock_irqsave(&conf->device_lock, flags); in raid5_start_reshape()
7388 mddev->degraded = calc_degraded(conf); in raid5_start_reshape()
7389 spin_unlock_irqrestore(&conf->device_lock, flags); in raid5_start_reshape()
7391 mddev->raid_disks = conf->raid_disks; in raid5_start_reshape()
7392 mddev->reshape_position = conf->reshape_progress; in raid5_start_reshape()
7404 spin_lock_irq(&conf->device_lock); in raid5_start_reshape()
7405 write_seqcount_begin(&conf->gen_lock); in raid5_start_reshape()
7406 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; in raid5_start_reshape()
7408 conf->chunk_sectors = conf->prev_chunk_sectors; in raid5_start_reshape()
7409 mddev->new_layout = conf->algorithm = conf->prev_algo; in raid5_start_reshape()
7413 conf->generation --; in raid5_start_reshape()
7414 conf->reshape_progress = MaxSector; in raid5_start_reshape()
7416 write_seqcount_end(&conf->gen_lock); in raid5_start_reshape()
7417 spin_unlock_irq(&conf->device_lock); in raid5_start_reshape()
7420 conf->reshape_checkpoint = jiffies; in raid5_start_reshape()
7429 static void end_reshape(struct r5conf *conf) in end_reshape() argument
7432 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { in end_reshape()
7435 spin_lock_irq(&conf->device_lock); in end_reshape()
7436 conf->previous_raid_disks = conf->raid_disks; in end_reshape()
7437 rdev_for_each(rdev, conf->mddev) in end_reshape()
7440 conf->reshape_progress = MaxSector; in end_reshape()
7441 spin_unlock_irq(&conf->device_lock); in end_reshape()
7442 wake_up(&conf->wait_for_overlap); in end_reshape()
7447 if (conf->mddev->queue) { in end_reshape()
7448 int data_disks = conf->raid_disks - conf->max_degraded; in end_reshape()
7449 int stripe = data_disks * ((conf->chunk_sectors << 9) in end_reshape()
7451 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) in end_reshape()
7452 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; in end_reshape()
7462 struct r5conf *conf = mddev->private; in raid5_finish_reshape() local
7472 spin_lock_irq(&conf->device_lock); in raid5_finish_reshape()
7473 mddev->degraded = calc_degraded(conf); in raid5_finish_reshape()
7474 spin_unlock_irq(&conf->device_lock); in raid5_finish_reshape()
7475 for (d = conf->raid_disks ; in raid5_finish_reshape()
7476 d < conf->raid_disks - mddev->delta_disks; in raid5_finish_reshape()
7478 struct md_rdev *rdev = conf->disks[d].rdev; in raid5_finish_reshape()
7481 rdev = conf->disks[d].replacement; in raid5_finish_reshape()
7486 mddev->layout = conf->algorithm; in raid5_finish_reshape()
7487 mddev->chunk_sectors = conf->chunk_sectors; in raid5_finish_reshape()
7496 struct r5conf *conf = mddev->private; in raid5_quiesce() local
7500 wake_up(&conf->wait_for_overlap); in raid5_quiesce()
7504 lock_all_device_hash_locks_irq(conf); in raid5_quiesce()
7508 conf->quiesce = 2; in raid5_quiesce()
7509 wait_event_cmd(conf->wait_for_stripe, in raid5_quiesce()
7510 atomic_read(&conf->active_stripes) == 0 && in raid5_quiesce()
7511 atomic_read(&conf->active_aligned_reads) == 0, in raid5_quiesce()
7512 unlock_all_device_hash_locks_irq(conf), in raid5_quiesce()
7513 lock_all_device_hash_locks_irq(conf)); in raid5_quiesce()
7514 conf->quiesce = 1; in raid5_quiesce()
7515 unlock_all_device_hash_locks_irq(conf); in raid5_quiesce()
7517 wake_up(&conf->wait_for_overlap); in raid5_quiesce()
7521 lock_all_device_hash_locks_irq(conf); in raid5_quiesce()
7522 conf->quiesce = 0; in raid5_quiesce()
7523 wake_up(&conf->wait_for_stripe); in raid5_quiesce()
7524 wake_up(&conf->wait_for_overlap); in raid5_quiesce()
7525 unlock_all_device_hash_locks_irq(conf); in raid5_quiesce()
7623 struct r5conf *conf = mddev->private; in raid5_check_reshape() local
7643 conf->algorithm = mddev->new_layout; in raid5_check_reshape()
7647 conf->chunk_sectors = new_chunk ; in raid5_check_reshape()