109 struct dio {  struct
156 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)  in dio_refill_pages()  argument
160 	ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,  in dio_refill_pages()
163 	if (ret < 0 && sdio->blocks_available && (dio->rw & WRITE)) {  in dio_refill_pages()
170 		if (dio->page_errors == 0)  in dio_refill_pages()
171 			dio->page_errors = ret;  in dio_refill_pages()
173 		dio->pages[0] = page;  in dio_refill_pages()
198 static inline struct page *dio_get_page(struct dio *dio,  in dio_get_page()  argument
204 		ret = dio_refill_pages(dio, sdio);  in dio_get_page()
209 	return dio->pages[sdio->head];  in dio_get_page()
224 static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret,  in dio_complete()  argument
238 	if (dio->result) {  in dio_complete()
239 		transferred = dio->result;  in dio_complete()
242 		if ((dio->rw == READ) && ((offset + transferred) > dio->i_size))  in dio_complete()
243 			transferred = dio->i_size - offset;  in dio_complete()
247 		ret = dio->page_errors;  in dio_complete()
249 		ret = dio->io_error;  in dio_complete()
253 	if (dio->end_io && dio->result)  in dio_complete()
254 		dio->end_io(dio->iocb, offset, transferred, dio->private);  in dio_complete()
256 	if (!(dio->flags & DIO_SKIP_DIO_COUNT))  in dio_complete()
257 		inode_dio_end(dio->inode);  in dio_complete()
260 		if (dio->rw & WRITE) {  in dio_complete()
263 			err = generic_write_sync(dio->iocb->ki_filp, offset,  in dio_complete()
269 		dio->iocb->ki_complete(dio->iocb, ret, 0);  in dio_complete()
272 	kmem_cache_free(dio_cache, dio);  in dio_complete()
278 	struct dio *dio = container_of(work, struct dio, complete_work);  in dio_aio_complete_work()  local
280 	dio_complete(dio, dio->iocb->ki_pos, 0, true);  in dio_aio_complete_work()
283 static int dio_bio_complete(struct dio *dio, struct bio *bio);
290 	struct dio *dio = bio->bi_private;  in dio_bio_end_aio()  local
295 	dio_bio_complete(dio, bio);  in dio_bio_end_aio()
297 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_end_aio()
298 	remaining = --dio->refcount;  in dio_bio_end_aio()
299 	if (remaining == 1 && dio->waiter)  in dio_bio_end_aio()
300 		wake_up_process(dio->waiter);  in dio_bio_end_aio()
301 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_end_aio()
304 		if (dio->result && dio->defer_completion) {  in dio_bio_end_aio()
305 			INIT_WORK(&dio->complete_work, dio_aio_complete_work);  in dio_bio_end_aio()
306 			queue_work(dio->inode->i_sb->s_dio_done_wq,  in dio_bio_end_aio()
307 				   &dio->complete_work);  in dio_bio_end_aio()
309 			dio_complete(dio, dio->iocb->ki_pos, 0, true);  in dio_bio_end_aio()
323 	struct dio *dio = bio->bi_private;  in dio_bio_end_io()  local
326 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_end_io()
327 	bio->bi_private = dio->bio_list;  in dio_bio_end_io()
328 	dio->bio_list = bio;  in dio_bio_end_io()
329 	if (--dio->refcount == 1 && dio->waiter)  in dio_bio_end_io()
330 		wake_up_process(dio->waiter);  in dio_bio_end_io()
331 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_end_io()
345 	struct dio *dio = bio->bi_private;  in dio_end_io()  local
347 	if (dio->is_async)  in dio_end_io()
355 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,  in dio_bio_alloc()  argument
369 	if (dio->is_async)  in dio_bio_alloc()
385 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)  in dio_bio_submit()  argument
390 	bio->bi_private = dio;  in dio_bio_submit()
392 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_submit()
393 	dio->refcount++;  in dio_bio_submit()
394 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_submit()
396 	if (dio->is_async && dio->rw == READ)  in dio_bio_submit()
400 		sdio->submit_io(dio->rw, bio, dio->inode,  in dio_bio_submit()
403 		submit_bio(dio->rw, bio);  in dio_bio_submit()
413 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)  in dio_cleanup()  argument
416 		page_cache_release(dio->pages[sdio->head++]);  in dio_cleanup()
425 static struct bio *dio_await_one(struct dio *dio)  in dio_await_one()  argument
430 	spin_lock_irqsave(&dio->bio_lock, flags);  in dio_await_one()
438 	while (dio->refcount > 1 && dio->bio_list == NULL) {  in dio_await_one()
440 		dio->waiter = current;  in dio_await_one()
441 		spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_await_one()
444 		spin_lock_irqsave(&dio->bio_lock, flags);  in dio_await_one()
445 		dio->waiter = NULL;  in dio_await_one()
447 	if (dio->bio_list) {  in dio_await_one()
448 		bio = dio->bio_list;  in dio_await_one()
449 		dio->bio_list = bio->bi_private;  in dio_await_one()
451 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_await_one()
458 static int dio_bio_complete(struct dio *dio, struct bio *bio)  in dio_bio_complete()  argument
465 		dio->io_error = -EIO;  in dio_bio_complete()
467 	if (dio->is_async && dio->rw == READ) {  in dio_bio_complete()
473 			if (dio->rw == READ && !PageCompound(page))  in dio_bio_complete()
489 static void dio_await_completion(struct dio *dio)  in dio_await_completion()  argument
493 		bio = dio_await_one(dio);  in dio_await_completion()
495 			dio_bio_complete(dio, bio);  in dio_await_completion()
506 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)  in dio_bio_reap()  argument
511 		while (dio->bio_list) {  in dio_bio_reap()
516 			spin_lock_irqsave(&dio->bio_lock, flags);  in dio_bio_reap()
517 			bio = dio->bio_list;  in dio_bio_reap()
518 			dio->bio_list = bio->bi_private;  in dio_bio_reap()
519 			spin_unlock_irqrestore(&dio->bio_lock, flags);  in dio_bio_reap()
520 			ret2 = dio_bio_complete(dio, bio);  in dio_bio_reap()
553 static int dio_set_defer_completion(struct dio *dio)  in dio_set_defer_completion()  argument
555 	struct super_block *sb = dio->inode->i_sb;  in dio_set_defer_completion()
557 	if (dio->defer_completion)  in dio_set_defer_completion()
559 	dio->defer_completion = true;  in dio_set_defer_completion()
588 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,  in get_more_blocks()  argument
602 	ret = dio->page_errors;  in get_more_blocks()
624 		create = dio->rw & WRITE;  in get_more_blocks()
625 		if (dio->flags & DIO_SKIP_HOLES) {  in get_more_blocks()
626 			if (sdio->block_in_file < (i_size_read(dio->inode) >>  in get_more_blocks()
631 		ret = (*sdio->get_block)(dio->inode, fs_startblk,  in get_more_blocks()
635 		dio->private = map_bh->b_private;  in get_more_blocks()
638 			ret = dio_set_defer_completion(dio);  in get_more_blocks()
646 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,  in dio_new_bio()  argument
652 	ret = dio_bio_reap(dio, sdio);  in dio_new_bio()
658 	dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);  in dio_new_bio()
703 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,  in dio_send_cur_page()  argument
729 			dio_bio_submit(dio, sdio);  in dio_send_cur_page()
733 		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);  in dio_send_cur_page()
739 		dio_bio_submit(dio, sdio);  in dio_send_cur_page()
740 		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);  in dio_send_cur_page()
768 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,  in submit_page_section()  argument
774 	if (dio->rw & WRITE) {  in submit_page_section()
796 		ret = dio_send_cur_page(dio, sdio, map_bh);  in submit_page_section()
815 		ret = dio_send_cur_page(dio, sdio, map_bh);  in submit_page_section()
816 		dio_bio_submit(dio, sdio);  in submit_page_section()
828 static void clean_blockdev_aliases(struct dio *dio, struct buffer_head *map_bh)  in clean_blockdev_aliases()  argument
833 	nblocks = map_bh->b_size >> dio->inode->i_blkbits;  in clean_blockdev_aliases()
850 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,  in dio_zero_block()  argument
878 	if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,  in dio_zero_block()
901 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,  in do_direct_IO()  argument
911 		page = dio_get_page(dio, sdio);  in do_direct_IO()
932 				ret = get_more_blocks(dio, sdio, map_bh);  in do_direct_IO()
945 					clean_blockdev_aliases(dio, map_bh);  in do_direct_IO()
974 				if (dio->rw & WRITE) {  in do_direct_IO()
983 				i_size_aligned = ALIGN(i_size_read(dio->inode),  in do_direct_IO()
994 				dio->result += 1 << blkbits;  in do_direct_IO()
1004 				dio_zero_block(dio, sdio, 0, map_bh);  in do_direct_IO()
1022 			ret = submit_page_section(dio, sdio, page,  in do_direct_IO()
1035 			dio->result += this_chunk_bytes;  in do_direct_IO()
1050 static inline int drop_refcount(struct dio *dio)  in drop_refcount()  argument
1066 	spin_lock_irqsave(&dio->bio_lock, flags);  in drop_refcount()
1067 	ret2 = --dio->refcount;  in drop_refcount()
1068 	spin_unlock_irqrestore(&dio->bio_lock, flags);  in drop_refcount()
1109 	struct dio *dio;  in do_blockdev_direct_IO()  local
1132 	dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);  in do_blockdev_direct_IO()
1134 	if (!dio)  in do_blockdev_direct_IO()
1141 	memset(dio, 0, offsetof(struct dio, pages));  in do_blockdev_direct_IO()
1143 	dio->flags = flags;  in do_blockdev_direct_IO()
1144 	if (dio->flags & DIO_LOCKING) {  in do_blockdev_direct_IO()
1156 				kmem_cache_free(dio_cache, dio);  in do_blockdev_direct_IO()
1163 	dio->i_size = i_size_read(inode);  in do_blockdev_direct_IO()
1164 	if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {  in do_blockdev_direct_IO()
1165 		if (dio->flags & DIO_LOCKING)  in do_blockdev_direct_IO()
1167 		kmem_cache_free(dio_cache, dio);  in do_blockdev_direct_IO()
1179 		dio->is_async = false;  in do_blockdev_direct_IO()
1180 	else if (!(dio->flags & DIO_ASYNC_EXTEND) &&  in do_blockdev_direct_IO()
1182 		dio->is_async = false;  in do_blockdev_direct_IO()
1184 		dio->is_async = true;  in do_blockdev_direct_IO()
1186 	dio->inode = inode;  in do_blockdev_direct_IO()
1187 	dio->rw = iov_iter_rw(iter) == WRITE ? WRITE_ODIRECT : READ;  in do_blockdev_direct_IO()
1193 	if (dio->is_async && iov_iter_rw(iter) == WRITE &&  in do_blockdev_direct_IO()
1196 		retval = dio_set_defer_completion(dio);  in do_blockdev_direct_IO()
1202 			kmem_cache_free(dio_cache, dio);  in do_blockdev_direct_IO()
1210 	if (!(dio->flags & DIO_SKIP_DIO_COUNT))  in do_blockdev_direct_IO()
1219 	dio->end_io = end_io;  in do_blockdev_direct_IO()
1224 	dio->iocb = iocb;  in do_blockdev_direct_IO()
1226 	spin_lock_init(&dio->bio_lock);  in do_blockdev_direct_IO()
1227 	dio->refcount = 1;  in do_blockdev_direct_IO()
1244 	retval = do_direct_IO(dio, &sdio, &map_bh);  in do_blockdev_direct_IO()
1246 		dio_cleanup(dio, &sdio);  in do_blockdev_direct_IO()
1259 	dio_zero_block(dio, &sdio, 1, &map_bh);  in do_blockdev_direct_IO()
1264 		ret2 = dio_send_cur_page(dio, &sdio, &map_bh);  in do_blockdev_direct_IO()
1271 		dio_bio_submit(dio, &sdio);  in do_blockdev_direct_IO()
1279 	dio_cleanup(dio, &sdio);  in do_blockdev_direct_IO()
1286 	if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING))  in do_blockdev_direct_IO()
1287 		mutex_unlock(&dio->inode->i_mutex);  in do_blockdev_direct_IO()
1297 	if (dio->is_async && retval == 0 && dio->result &&  in do_blockdev_direct_IO()
1298 	    (iov_iter_rw(iter) == READ || dio->result == count))  in do_blockdev_direct_IO()
1301 		dio_await_completion(dio);  in do_blockdev_direct_IO()
1303 	if (drop_refcount(dio) == 0) {  in do_blockdev_direct_IO()
1304 		retval = dio_complete(dio, offset, retval, false);  in do_blockdev_direct_IO()
1338 	dio_cache = KMEM_CACHE(dio, SLAB_PANIC);  in dio_init()