381 	struct dm_cache_migration *mg;  member
408 	struct dm_cache_migration *mg;  in alloc_migration()  local
410 	mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);  in alloc_migration()
411 	if (mg) {  in alloc_migration()
412 		mg->cache = cache;  in alloc_migration()
413 		atomic_inc(&mg->cache->nr_allocated_migrations);  in alloc_migration()
416 	return mg;  in alloc_migration()
419 static void free_migration(struct dm_cache_migration *mg)  in free_migration()  argument
421 	struct cache *cache = mg->cache;  in free_migration()
426 	mempool_free(mg, cache->migration_pool);  in free_migration()
431 	if (!p->mg) {  in prealloc_data_structs()
432 		p->mg = alloc_migration(cache);  in prealloc_data_structs()
433 		if (!p->mg)  in prealloc_data_structs()
460 	if (p->mg)  in prealloc_free_structs()
461 		free_migration(p->mg);  in prealloc_free_structs()
466 	struct dm_cache_migration *mg = p->mg;  in prealloc_get_migration()  local
468 	BUG_ON(!mg);  in prealloc_get_migration()
469 	p->mg = NULL;  in prealloc_get_migration()
471 	return mg;  in prealloc_get_migration()
1119 static void free_io_migration(struct dm_cache_migration *mg)  in free_io_migration()  argument
1121 	struct cache *cache = mg->cache;  in free_io_migration()
1124 	free_migration(mg);  in free_io_migration()
1128 static void migration_failure(struct dm_cache_migration *mg)  in migration_failure()  argument
1130 	struct cache *cache = mg->cache;  in migration_failure()
1133 	if (mg->writeback) {  in migration_failure()
1135 		set_dirty(cache, mg->old_oblock, mg->cblock);  in migration_failure()
1136 		cell_defer(cache, mg->old_ocell, false);  in migration_failure()
1138 	} else if (mg->demote) {  in migration_failure()
1140 		policy_force_mapping(cache->policy, mg->new_oblock, mg->old_oblock);  in migration_failure()
1142 		cell_defer(cache, mg->old_ocell, mg->promote ? false : true);  in migration_failure()
1143 		if (mg->promote)  in migration_failure()
1144 			cell_defer(cache, mg->new_ocell, true);  in migration_failure()
1147 		policy_remove_mapping(cache->policy, mg->new_oblock);  in migration_failure()
1148 		cell_defer(cache, mg->new_ocell, true);  in migration_failure()
1151 	free_io_migration(mg);  in migration_failure()
1154 static void migration_success_pre_commit(struct dm_cache_migration *mg)  in migration_success_pre_commit()  argument
1158 	struct cache *cache = mg->cache;  in migration_success_pre_commit()
1160 	if (mg->writeback) {  in migration_success_pre_commit()
1161 		clear_dirty(cache, mg->old_oblock, mg->cblock);  in migration_success_pre_commit()
1162 		cell_defer(cache, mg->old_ocell, false);  in migration_success_pre_commit()
1163 		free_io_migration(mg);  in migration_success_pre_commit()
1166 	} else if (mg->demote) {  in migration_success_pre_commit()
1167 		r = dm_cache_remove_mapping(cache->cmd, mg->cblock);  in migration_success_pre_commit()
1172 			policy_force_mapping(cache->policy, mg->new_oblock,  in migration_success_pre_commit()
1173 					     mg->old_oblock);  in migration_success_pre_commit()
1174 			if (mg->promote)  in migration_success_pre_commit()
1175 				cell_defer(cache, mg->new_ocell, true);  in migration_success_pre_commit()
1176 			free_io_migration(mg);  in migration_success_pre_commit()
1180 		r = dm_cache_insert_mapping(cache->cmd, mg->cblock, mg->new_oblock);  in migration_success_pre_commit()
1185 			policy_remove_mapping(cache->policy, mg->new_oblock);  in migration_success_pre_commit()
1186 			free_io_migration(mg);  in migration_success_pre_commit()
1192 	list_add_tail(&mg->list, &cache->need_commit_migrations);  in migration_success_pre_commit()
1197 static void migration_success_post_commit(struct dm_cache_migration *mg)  in migration_success_post_commit()  argument
1200 	struct cache *cache = mg->cache;  in migration_success_post_commit()
1202 	if (mg->writeback) {  in migration_success_post_commit()
1207 	} else if (mg->demote) {  in migration_success_post_commit()
1208 		cell_defer(cache, mg->old_ocell, mg->promote ? false : true);  in migration_success_post_commit()
1210 		if (mg->promote) {  in migration_success_post_commit()
1211 			mg->demote = false;  in migration_success_post_commit()
1214 			list_add_tail(&mg->list, &cache->quiesced_migrations);  in migration_success_post_commit()
1218 			if (mg->invalidate)  in migration_success_post_commit()
1219 				policy_remove_mapping(cache->policy, mg->old_oblock);  in migration_success_post_commit()
1220 			free_io_migration(mg);  in migration_success_post_commit()
1224 		if (mg->requeue_holder) {  in migration_success_post_commit()
1225 			clear_dirty(cache, mg->new_oblock, mg->cblock);  in migration_success_post_commit()
1226 			cell_defer(cache, mg->new_ocell, true);  in migration_success_post_commit()
1231 			set_dirty(cache, mg->new_oblock, mg->cblock);  in migration_success_post_commit()
1232 			bio_endio(mg->new_ocell->holder);  in migration_success_post_commit()
1233 			cell_defer(cache, mg->new_ocell, false);  in migration_success_post_commit()
1235 		free_io_migration(mg);  in migration_success_post_commit()
1242 	struct dm_cache_migration *mg = (struct dm_cache_migration *) context;  in copy_complete()  local
1243 	struct cache *cache = mg->cache;  in copy_complete()
1246 		mg->err = true;  in copy_complete()
1249 	list_add_tail(&mg->list, &cache->completed_migrations);  in copy_complete()
1255 static void issue_copy(struct dm_cache_migration *mg)  in issue_copy()  argument
1259 	struct cache *cache = mg->cache;  in issue_copy()
1260 	sector_t cblock = from_cblock(mg->cblock);  in issue_copy()
1269 	if (mg->writeback || mg->demote) {  in issue_copy()
1271 		o_region.sector = from_oblock(mg->old_oblock) * cache->sectors_per_block;  in issue_copy()
1272 		r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, mg);  in issue_copy()
1275 		o_region.sector = from_oblock(mg->new_oblock) * cache->sectors_per_block;  in issue_copy()
1276 		r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, mg);  in issue_copy()
1281 		migration_failure(mg);  in issue_copy()
1287 	struct dm_cache_migration *mg = bio->bi_private;  in overwrite_endio()  local
1288 	struct cache *cache = mg->cache;  in overwrite_endio()
1296 		mg->err = true;  in overwrite_endio()
1298 	mg->requeue_holder = false;  in overwrite_endio()
1301 	list_add_tail(&mg->list, &cache->completed_migrations);  in overwrite_endio()
1307 static void issue_overwrite(struct dm_cache_migration *mg, struct bio *bio)  in issue_overwrite()  argument
1309 	size_t pb_data_size = get_per_bio_data_size(mg->cache);  in issue_overwrite()
1312 	dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);  in issue_overwrite()
1313 	remap_to_cache_dirty(mg->cache, bio, mg->new_oblock, mg->cblock);  in issue_overwrite()
1319 	accounted_request(mg->cache, bio);  in issue_overwrite()
1328 static void avoid_copy(struct dm_cache_migration *mg)  in avoid_copy()  argument
1330 	atomic_inc(&mg->cache->stats.copies_avoided);  in avoid_copy()
1331 	migration_success_pre_commit(mg);  in avoid_copy()
1348 static void issue_discard(struct dm_cache_migration *mg)  in issue_discard()  argument
1351 	struct bio *bio = mg->new_ocell->holder;  in issue_discard()
1352 	struct cache *cache = mg->cache;  in issue_discard()
1361 	cell_defer(cache, mg->new_ocell, false);  in issue_discard()
1362 	free_migration(mg);  in issue_discard()
1366 static void issue_copy_or_discard(struct dm_cache_migration *mg)  in issue_copy_or_discard()  argument
1369 	struct cache *cache = mg->cache;  in issue_copy_or_discard()
1371 	if (mg->discard) {  in issue_copy_or_discard()
1372 		issue_discard(mg);  in issue_copy_or_discard()
1376 	if (mg->writeback || mg->demote)  in issue_copy_or_discard()
1377 		avoid = !is_dirty(cache, mg->cblock) ||  in issue_copy_or_discard()
1378 			is_discarded_oblock(cache, mg->old_oblock);  in issue_copy_or_discard()
1380 		struct bio *bio = mg->new_ocell->holder;  in issue_copy_or_discard()
1382 		avoid = is_discarded_oblock(cache, mg->new_oblock);  in issue_copy_or_discard()
1386 			issue_overwrite(mg, bio);  in issue_copy_or_discard()
1391 	avoid ? avoid_copy(mg) : issue_copy(mg);  in issue_copy_or_discard()
1394 static void complete_migration(struct dm_cache_migration *mg)  in complete_migration()  argument
1396 	if (mg->err)  in complete_migration()
1397 		migration_failure(mg);  in complete_migration()
1399 		migration_success_pre_commit(mg);  in complete_migration()
1407 	struct dm_cache_migration *mg, *tmp;  in process_migrations()  local
1414 	list_for_each_entry_safe(mg, tmp, &list, list)  in process_migrations()
1415 		fn(mg);  in process_migrations()
1418 static void __queue_quiesced_migration(struct dm_cache_migration *mg)  in __queue_quiesced_migration()  argument
1420 	list_add_tail(&mg->list, &mg->cache->quiesced_migrations);  in __queue_quiesced_migration()
1423 static void queue_quiesced_migration(struct dm_cache_migration *mg)  in queue_quiesced_migration()  argument
1426 	struct cache *cache = mg->cache;  in queue_quiesced_migration()
1429 	__queue_quiesced_migration(mg);  in queue_quiesced_migration()
1438 	struct dm_cache_migration *mg, *tmp;  in queue_quiesced_migrations()  local
1441 	list_for_each_entry_safe(mg, tmp, work, list)  in queue_quiesced_migrations()
1442 		__queue_quiesced_migration(mg);  in queue_quiesced_migrations()
1463 static void quiesce_migration(struct dm_cache_migration *mg)  in quiesce_migration()  argument
1465 	if (!dm_deferred_set_add_work(mg->cache->all_io_ds, &mg->list))  in quiesce_migration()
1466 		queue_quiesced_migration(mg);  in quiesce_migration()
1473 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in promote()  local
1475 	mg->err = false;  in promote()
1476 	mg->discard = false;  in promote()
1477 	mg->writeback = false;  in promote()
1478 	mg->demote = false;  in promote()
1479 	mg->promote = true;  in promote()
1480 	mg->requeue_holder = true;  in promote()
1481 	mg->invalidate = false;  in promote()
1482 	mg->cache = cache;  in promote()
1483 	mg->new_oblock = oblock;  in promote()
1484 	mg->cblock = cblock;  in promote()
1485 	mg->old_ocell = NULL;  in promote()
1486 	mg->new_ocell = cell;  in promote()
1487 	mg->start_jiffies = jiffies;  in promote()
1490 	quiesce_migration(mg);  in promote()
1497 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in writeback()  local
1499 	mg->err = false;  in writeback()
1500 	mg->discard = false;  in writeback()
1501 	mg->writeback = true;  in writeback()
1502 	mg->demote = false;  in writeback()
1503 	mg->promote = false;  in writeback()
1504 	mg->requeue_holder = true;  in writeback()
1505 	mg->invalidate = false;  in writeback()
1506 	mg->cache = cache;  in writeback()
1507 	mg->old_oblock = oblock;  in writeback()
1508 	mg->cblock = cblock;  in writeback()
1509 	mg->old_ocell = cell;  in writeback()
1510 	mg->new_ocell = NULL;  in writeback()
1511 	mg->start_jiffies = jiffies;  in writeback()
1514 	quiesce_migration(mg);  in writeback()
1523 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in demote_then_promote()  local
1525 	mg->err = false;  in demote_then_promote()
1526 	mg->discard = false;  in demote_then_promote()
1527 	mg->writeback = false;  in demote_then_promote()
1528 	mg->demote = true;  in demote_then_promote()
1529 	mg->promote = true;  in demote_then_promote()
1530 	mg->requeue_holder = true;  in demote_then_promote()
1531 	mg->invalidate = false;  in demote_then_promote()
1532 	mg->cache = cache;  in demote_then_promote()
1533 	mg->old_oblock = old_oblock;  in demote_then_promote()
1534 	mg->new_oblock = new_oblock;  in demote_then_promote()
1535 	mg->cblock = cblock;  in demote_then_promote()
1536 	mg->old_ocell = old_ocell;  in demote_then_promote()
1537 	mg->new_ocell = new_ocell;  in demote_then_promote()
1538 	mg->start_jiffies = jiffies;  in demote_then_promote()
1541 	quiesce_migration(mg);  in demote_then_promote()
1552 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in invalidate()  local
1554 	mg->err = false;  in invalidate()
1555 	mg->discard = false;  in invalidate()
1556 	mg->writeback = false;  in invalidate()
1557 	mg->demote = true;  in invalidate()
1558 	mg->promote = false;  in invalidate()
1559 	mg->requeue_holder = true;  in invalidate()
1560 	mg->invalidate = true;  in invalidate()
1561 	mg->cache = cache;  in invalidate()
1562 	mg->old_oblock = oblock;  in invalidate()
1563 	mg->cblock = cblock;  in invalidate()
1564 	mg->old_ocell = cell;  in invalidate()
1565 	mg->new_ocell = NULL;  in invalidate()
1566 	mg->start_jiffies = jiffies;  in invalidate()
1569 	quiesce_migration(mg);  in invalidate()
1575 	struct dm_cache_migration *mg = prealloc_get_migration(structs);  in discard()  local
1577 	mg->err = false;  in discard()
1578 	mg->discard = true;  in discard()
1579 	mg->writeback = false;  in discard()
1580 	mg->demote = false;  in discard()
1581 	mg->promote = false;  in discard()
1582 	mg->requeue_holder = false;  in discard()
1583 	mg->invalidate = false;  in discard()
1584 	mg->cache = cache;  in discard()
1585 	mg->old_ocell = NULL;  in discard()
1586 	mg->new_ocell = cell;  in discard()
1587 	mg->start_jiffies = jiffies;  in discard()
1589 	quiesce_migration(mg);  in discard()