153 static void r5l_free_io_unit(struct r5l_log *log, struct r5l_io_unit *io)  in r5l_free_io_unit()  argument
155 	__free_page(io->meta_page);  in r5l_free_io_unit()
156 	kmem_cache_free(log->io_kc, io);  in r5l_free_io_unit()
162 	struct r5l_io_unit *io;  in r5l_move_io_unit_list()  local
165 		io = list_first_entry(from, struct r5l_io_unit, log_sibling);  in r5l_move_io_unit_list()
167 		if (io->state >= state)  in r5l_move_io_unit_list()
168 			list_move_tail(&io->log_sibling, to);  in r5l_move_io_unit_list()
174 static void __r5l_set_io_unit_state(struct r5l_io_unit *io,  in __r5l_set_io_unit_state()  argument
177 	if (WARN_ON(io->state >= state))  in __r5l_set_io_unit_state()
179 	io->state = state;  in __r5l_set_io_unit_state()
182 static void r5l_io_run_stripes(struct r5l_io_unit *io)  in r5l_io_run_stripes()  argument
186 	list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {  in r5l_io_run_stripes()
195 	struct r5l_io_unit *io, *next;  in r5l_log_run_stripes()  local
199 	list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {  in r5l_log_run_stripes()
201 		if (io->state < IO_UNIT_IO_END)  in r5l_log_run_stripes()
204 		list_move_tail(&io->log_sibling, &log->finished_ios);  in r5l_log_run_stripes()
205 		r5l_io_run_stripes(io);  in r5l_log_run_stripes()
211 	struct r5l_io_unit *io = bio->bi_private;  in r5l_log_endio()  local
212 	struct r5l_log *log = io->log;  in r5l_log_endio()
221 	__r5l_set_io_unit_state(io, IO_UNIT_IO_END);  in r5l_log_endio()
235 	struct r5l_io_unit *io = log->current_io;  in r5l_submit_current_io()  local
240 	if (!io)  in r5l_submit_current_io()
243 	block = page_address(io->meta_page);  in r5l_submit_current_io()
244 	block->meta_size = cpu_to_le32(io->meta_offset);  in r5l_submit_current_io()
250 	__r5l_set_io_unit_state(io, IO_UNIT_IO_START);  in r5l_submit_current_io()
253 	submit_bio(WRITE, io->current_bio);  in r5l_submit_current_io()
267 static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)  in r5_reserve_log_entry()  argument
279 		io->need_split_bio = true;  in r5_reserve_log_entry()
281 	io->log_end = log->log_start;  in r5_reserve_log_entry()
286 	struct r5l_io_unit *io;  in r5l_new_meta()  local
290 	io = kmem_cache_zalloc(log->io_kc, GFP_NOIO | __GFP_NOFAIL);  in r5l_new_meta()
291 	io->log = log;  in r5l_new_meta()
292 	INIT_LIST_HEAD(&io->log_sibling);  in r5l_new_meta()
293 	INIT_LIST_HEAD(&io->stripe_list);  in r5l_new_meta()
294 	io->state = IO_UNIT_RUNNING;  in r5l_new_meta()
296 	io->meta_page = alloc_page(GFP_NOIO | __GFP_NOFAIL | __GFP_ZERO);  in r5l_new_meta()
297 	block = page_address(io->meta_page);  in r5l_new_meta()
303 	io->log_start = log->log_start;  in r5l_new_meta()
304 	io->meta_offset = sizeof(struct r5l_meta_block);  in r5l_new_meta()
305 	io->seq = log->seq++;  in r5l_new_meta()
307 	io->current_bio = r5l_bio_alloc(log);  in r5l_new_meta()
308 	io->current_bio->bi_end_io = r5l_log_endio;  in r5l_new_meta()
309 	io->current_bio->bi_private = io;  in r5l_new_meta()
310 	bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0);  in r5l_new_meta()
312 	r5_reserve_log_entry(log, io);  in r5l_new_meta()
315 	list_add_tail(&io->log_sibling, &log->running_ios);  in r5l_new_meta()
318 	return io;  in r5l_new_meta()
337 	struct r5l_io_unit *io = log->current_io;  in r5l_append_payload_meta()  local
340 	payload = page_address(io->meta_page) + io->meta_offset;  in r5l_append_payload_meta()
350 	io->meta_offset += sizeof(struct r5l_payload_data_parity) +  in r5l_append_payload_meta()
356 	struct r5l_io_unit *io = log->current_io;  in r5l_append_payload_page()  local
358 	if (io->need_split_bio) {  in r5l_append_payload_page()
359 		struct bio *prev = io->current_bio;  in r5l_append_payload_page()
361 		io->current_bio = r5l_bio_alloc(log);  in r5l_append_payload_page()
362 		bio_chain(io->current_bio, prev);  in r5l_append_payload_page()
367 	if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))  in r5l_append_payload_page()
370 	r5_reserve_log_entry(log, io);  in r5l_append_payload_page()
378 	struct r5l_io_unit *io;  in r5l_log_stripe()  local
387 	io = log->current_io;  in r5l_log_stripe()
413 	list_add_tail(&sh->log_list, &io->stripe_list);  in r5l_log_stripe()
414 	atomic_inc(&io->pending_stripe);  in r5l_log_stripe()
415 	sh->log_io = io;  in r5l_log_stripe()
543 	struct r5l_io_unit *io, *next;  in r5l_complete_finished_ios()  local
548 	list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) {  in r5l_complete_finished_ios()
550 		if (io->state < IO_UNIT_STRIPE_END)  in r5l_complete_finished_ios()
553 		log->next_checkpoint = io->log_start;  in r5l_complete_finished_ios()
554 		log->next_cp_seq = io->seq;  in r5l_complete_finished_ios()
556 		list_del(&io->log_sibling);  in r5l_complete_finished_ios()
557 		r5l_free_io_unit(log, io);  in r5l_complete_finished_ios()
565 static void __r5l_stripe_write_finished(struct r5l_io_unit *io)  in __r5l_stripe_write_finished()  argument
567 	struct r5l_log *log = io->log;  in __r5l_stripe_write_finished()
571 	__r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END);  in __r5l_stripe_write_finished()
587 	struct r5l_io_unit *io;  in r5l_stripe_write_finished()  local
589 	io = sh->log_io;  in r5l_stripe_write_finished()
592 	if (io && atomic_dec_and_test(&io->pending_stripe))  in r5l_stripe_write_finished()
593 		__r5l_stripe_write_finished(io);  in r5l_stripe_write_finished()
601 	struct r5l_io_unit *io;  in r5l_log_flush_endio()  local
607 	list_for_each_entry(io, &log->flushing_ios, log_sibling)  in r5l_log_flush_endio()
608 		r5l_io_run_stripes(io);  in r5l_log_flush_endio()