131 static void put_pending_block(struct log_writes_c *lc)  in put_pending_block()  argument
133 	if (atomic_dec_and_test(&lc->pending_blocks)) {  in put_pending_block()
135 		if (waitqueue_active(&lc->wait))  in put_pending_block()
136 			wake_up(&lc->wait);  in put_pending_block()
140 static void put_io_block(struct log_writes_c *lc)  in put_io_block()  argument
142 	if (atomic_dec_and_test(&lc->io_blocks)) {  in put_io_block()
144 		if (waitqueue_active(&lc->wait))  in put_io_block()
145 			wake_up(&lc->wait);  in put_io_block()
151 	struct log_writes_c *lc = bio->bi_private;  in log_end_io()  local
159 		spin_lock_irqsave(&lc->blocks_lock, flags);  in log_end_io()
160 		lc->logging_enabled = false;  in log_end_io()
161 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in log_end_io()
167 	put_io_block(lc);  in log_end_io()
175 static void free_pending_block(struct log_writes_c *lc,  in free_pending_block()  argument
186 	put_pending_block(lc);  in free_pending_block()
189 static int write_metadata(struct log_writes_c *lc, void *entry,  in write_metadata()  argument
205 	bio->bi_bdev = lc->logdev->bdev;  in write_metadata()
207 	bio->bi_private = lc;  in write_metadata()
222 	       lc->sectorsize - entrylen - datalen);  in write_metadata()
225 	ret = bio_add_page(bio, page, lc->sectorsize, 0);  in write_metadata()
226 	if (ret != lc->sectorsize) {  in write_metadata()
236 	put_io_block(lc);  in write_metadata()
240 static int log_one_block(struct log_writes_c *lc,  in log_one_block()  argument
252 	if (write_metadata(lc, &entry, sizeof(entry), block->data,  in log_one_block()
254 		free_pending_block(lc, block);  in log_one_block()
267 	atomic_inc(&lc->io_blocks);  in log_one_block()
270 	bio->bi_bdev = lc->logdev->bdev;  in log_one_block()
272 	bio->bi_private = lc;  in log_one_block()
283 			atomic_inc(&lc->io_blocks);  in log_one_block()
292 			bio->bi_bdev = lc->logdev->bdev;  in log_one_block()
294 			bio->bi_private = lc;  in log_one_block()
311 	put_pending_block(lc);  in log_one_block()
314 	free_pending_block(lc, block);  in log_one_block()
315 	put_io_block(lc);  in log_one_block()
319 static int log_super(struct log_writes_c *lc)  in log_super()  argument
325 	super.nr_entries = cpu_to_le64(lc->logged_entries);  in log_super()
326 	super.sectorsize = cpu_to_le32(lc->sectorsize);  in log_super()
328 	if (write_metadata(lc, &super, sizeof(super), NULL, 0, 0)) {  in log_super()
336 static inline sector_t logdev_last_sector(struct log_writes_c *lc)  in logdev_last_sector()  argument
338 	return i_size_read(lc->logdev->bdev->bd_inode) >> SECTOR_SHIFT;  in logdev_last_sector()
343 	struct log_writes_c *lc = (struct log_writes_c *)arg;  in log_writes_kthread()  local
352 		spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
353 		if (!list_empty(&lc->logging_blocks)) {  in log_writes_kthread()
354 			block = list_first_entry(&lc->logging_blocks,  in log_writes_kthread()
357 			if (!lc->logging_enabled)  in log_writes_kthread()
360 			sector = lc->next_sector;  in log_writes_kthread()
362 				lc->next_sector++;  in log_writes_kthread()
364 				lc->next_sector += block->nr_sectors + 1;  in log_writes_kthread()
370 			if (!lc->end_sector)  in log_writes_kthread()
371 				lc->end_sector = logdev_last_sector(lc);  in log_writes_kthread()
372 			if (lc->end_sector &&  in log_writes_kthread()
373 			    lc->next_sector >= lc->end_sector) {  in log_writes_kthread()
375 				lc->logging_enabled = false;  in log_writes_kthread()
378 			lc->logged_entries++;  in log_writes_kthread()
379 			atomic_inc(&lc->io_blocks);  in log_writes_kthread()
383 				atomic_inc(&lc->io_blocks);  in log_writes_kthread()
386 		logging_enabled = lc->logging_enabled;  in log_writes_kthread()
387 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
390 				ret = log_one_block(lc, block, sector);  in log_writes_kthread()
392 					ret = log_super(lc);  in log_writes_kthread()
394 					spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
395 					lc->logging_enabled = false;  in log_writes_kthread()
396 					spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
399 				free_pending_block(lc, block);  in log_writes_kthread()
406 			    !atomic_read(&lc->pending_blocks))  in log_writes_kthread()
420 	struct log_writes_c *lc;  in log_writes_ctr()  local
432 	lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);  in log_writes_ctr()
433 	if (!lc) {  in log_writes_ctr()
437 	spin_lock_init(&lc->blocks_lock);  in log_writes_ctr()
438 	INIT_LIST_HEAD(&lc->unflushed_blocks);  in log_writes_ctr()
439 	INIT_LIST_HEAD(&lc->logging_blocks);  in log_writes_ctr()
440 	init_waitqueue_head(&lc->wait);  in log_writes_ctr()
441 	lc->sectorsize = 1 << SECTOR_SHIFT;  in log_writes_ctr()
442 	atomic_set(&lc->io_blocks, 0);  in log_writes_ctr()
443 	atomic_set(&lc->pending_blocks, 0);  in log_writes_ctr()
446 	if (dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev)) {  in log_writes_ctr()
452 	if (dm_get_device(ti, logdevname, dm_table_get_mode(ti->table), &lc->logdev)) {  in log_writes_ctr()
454 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
458 	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");  in log_writes_ctr()
459 	if (!lc->log_kthread) {  in log_writes_ctr()
461 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
462 		dm_put_device(ti, lc->logdev);  in log_writes_ctr()
467 	lc->next_sector = 1;  in log_writes_ctr()
468 	lc->logging_enabled = true;  in log_writes_ctr()
469 	lc->end_sector = logdev_last_sector(lc);  in log_writes_ctr()
470 	lc->device_supports_discard = true;  in log_writes_ctr()
477 	ti->private = lc;  in log_writes_ctr()
481 	kfree(lc);  in log_writes_ctr()
485 static int log_mark(struct log_writes_c *lc, char *data)  in log_mark()  argument
488 	size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);  in log_mark()
502 	atomic_inc(&lc->pending_blocks);  in log_mark()
505 	spin_lock_irq(&lc->blocks_lock);  in log_mark()
506 	list_add_tail(&block->list, &lc->logging_blocks);  in log_mark()
507 	spin_unlock_irq(&lc->blocks_lock);  in log_mark()
508 	wake_up_process(lc->log_kthread);  in log_mark()
514 	struct log_writes_c *lc = ti->private;  in log_writes_dtr()  local
516 	spin_lock_irq(&lc->blocks_lock);  in log_writes_dtr()
517 	list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);  in log_writes_dtr()
518 	spin_unlock_irq(&lc->blocks_lock);  in log_writes_dtr()
524 	log_mark(lc, "dm-log-writes-end");  in log_writes_dtr()
525 	wake_up_process(lc->log_kthread);  in log_writes_dtr()
526 	wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&  in log_writes_dtr()
527 		   !atomic_read(&lc->pending_blocks));  in log_writes_dtr()
528 	kthread_stop(lc->log_kthread);  in log_writes_dtr()
530 	WARN_ON(!list_empty(&lc->logging_blocks));  in log_writes_dtr()
531 	WARN_ON(!list_empty(&lc->unflushed_blocks));  in log_writes_dtr()
532 	dm_put_device(ti, lc->dev);  in log_writes_dtr()
533 	dm_put_device(ti, lc->logdev);  in log_writes_dtr()
534 	kfree(lc);  in log_writes_dtr()
539 	struct log_writes_c *lc = ti->private;  in normal_map_bio()  local
541 	bio->bi_bdev = lc->dev->bdev;  in normal_map_bio()
546 	struct log_writes_c *lc = ti->private;  in log_writes_map()  local
560 	if (!lc->logging_enabled)  in log_writes_map()
585 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
586 		lc->logging_enabled = false;  in log_writes_map()
587 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
592 	atomic_inc(&lc->pending_blocks);  in log_writes_map()
607 		if (lc->device_supports_discard)  in log_writes_map()
615 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
616 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
617 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
637 			free_pending_block(lc, block);  in log_writes_map()
638 			spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
639 			lc->logging_enabled = false;  in log_writes_map()
640 			spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
657 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
658 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
659 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
668 	struct log_writes_c *lc = ti->private;  in normal_end_io()  local
675 		spin_lock_irqsave(&lc->blocks_lock, flags);  in normal_end_io()
677 			list_splice_tail_init(&block->list, &lc->logging_blocks);  in normal_end_io()
678 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
679 			wake_up_process(lc->log_kthread);  in normal_end_io()
681 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
682 			wake_up_process(lc->log_kthread);  in normal_end_io()
684 			list_add_tail(&block->list, &lc->unflushed_blocks);  in normal_end_io()
685 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in normal_end_io()
699 	struct log_writes_c *lc = ti->private;  in log_writes_status()  local
703 		DMEMIT("%llu %llu", lc->logged_entries,  in log_writes_status()
704 		       (unsigned long long)lc->next_sector - 1);  in log_writes_status()
705 		if (!lc->logging_enabled)  in log_writes_status()
710 		DMEMIT("%s %s", lc->dev->name, lc->logdev->name);  in log_writes_status()
718 	struct log_writes_c *lc = ti->private;  in log_writes_ioctl()  local
719 	struct dm_dev *dev = lc->dev;  in log_writes_ioctl()
734 	struct log_writes_c *lc = ti->private;  in log_writes_merge()  local
735 	struct request_queue *q = bdev_get_queue(lc->dev->bdev);  in log_writes_merge()
740 	bvm->bi_bdev = lc->dev->bdev;  in log_writes_merge()
750 	struct log_writes_c *lc = ti->private;  in log_writes_iterate_devices()  local
752 	return fn(ti, lc->dev, 0, ti->len, data);  in log_writes_iterate_devices()
762 	struct log_writes_c *lc = ti->private;  in log_writes_message()  local
770 		r = log_mark(lc, argv[1]);  in log_writes_message()
779 	struct log_writes_c *lc = ti->private;  in log_writes_io_hints()  local
780 	struct request_queue *q = bdev_get_queue(lc->dev->bdev);  in log_writes_io_hints()
783 		lc->device_supports_discard = false;  in log_writes_io_hints()