55 static void bch_data_insert_keys(struct closure *cl)  in bch_data_insert_keys()  argument
57 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_keys()
70 	while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)  in bch_data_insert_keys()
71 		closure_sync(&s->cl);  in bch_data_insert_keys()
76 					  op->flush_journal ? cl : NULL);  in bch_data_insert_keys()
91 		continue_at(cl, bch_data_insert_start, op->wq);  in bch_data_insert_keys()
94 	closure_return(cl);  in bch_data_insert_keys()
115 static void bch_data_invalidate(struct closure *cl)  in bch_data_invalidate()  argument
117 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_invalidate()
140 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_invalidate()
143 static void bch_data_insert_error(struct closure *cl)  in bch_data_insert_error()  argument
145 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_error()
170 	bch_data_insert_keys(cl);  in bch_data_insert_error()
175 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()  local
176 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_endio()
183 			set_closure_fn(cl, bch_data_insert_error, op->wq);  in bch_data_insert_endio()
185 			set_closure_fn(cl, NULL, NULL);  in bch_data_insert_endio()
191 static void bch_data_insert_start(struct closure *cl)  in bch_data_insert_start()  argument
193 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_start()
202 		return bch_data_invalidate(cl);  in bch_data_insert_start()
219 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
234 		n->bi_private	= cl;  in bch_data_insert_start()
256 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
275 		return bch_data_invalidate(cl);  in bch_data_insert_start()
285 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
287 			closure_return(cl);  in bch_data_insert_start()
310 void bch_data_insert(struct closure *cl)  in bch_data_insert()  argument
312 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert()
319 	bch_data_insert_start(cl);  in bch_data_insert()
456 	struct closure		cl;  member
477 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()  local
478 	struct search *s = container_of(cl, struct search, cl);  in bch_cache_read_endio()
550 	n->bi_private	= &s->cl;  in cache_lookup_fn()
567 static void cache_lookup(struct closure *cl)  in cache_lookup()  argument
569 	struct search *s = container_of(cl, struct search, iop.cl);  in cache_lookup()
579 		continue_at(cl, cache_lookup, bcache_wq);  in cache_lookup()
581 	closure_return(cl);  in cache_lookup()
588 	struct closure *cl = bio->bi_private;  in request_endio()  local
591 		struct search *s = container_of(cl, struct search, cl);  in request_endio()
598 	closure_put(cl);  in request_endio()
620 	bio->bi_private		= &s->cl;  in do_bio_hook()
625 static void search_free(struct closure *cl)  in search_free()  argument
627 	struct search *s = container_of(cl, struct search, cl);  in search_free()
633 	closure_debug_destroy(cl);  in search_free()
644 	closure_init(&s->cl, NULL);  in search_alloc()
670 static void cached_dev_bio_complete(struct closure *cl)  in cached_dev_bio_complete()  argument
672 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_bio_complete()
675 	search_free(cl);  in cached_dev_bio_complete()
681 static void cached_dev_cache_miss_done(struct closure *cl)  in cached_dev_cache_miss_done()  argument
683 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_cache_miss_done()
696 	cached_dev_bio_complete(cl);  in cached_dev_cache_miss_done()
699 static void cached_dev_read_error(struct closure *cl)  in cached_dev_read_error()  argument
701 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error()
713 		closure_bio_submit(bio, cl, s->d);  in cached_dev_read_error()
716 	continue_at(cl, cached_dev_cache_miss_done, NULL);  in cached_dev_read_error()
719 static void cached_dev_read_done(struct closure *cl)  in cached_dev_read_done()  argument
721 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done()
753 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_read_done()
756 	continue_at(cl, cached_dev_cache_miss_done, NULL);  in cached_dev_read_done()
759 static void cached_dev_read_done_bh(struct closure *cl)  in cached_dev_read_done_bh()  argument
761 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done_bh()
769 		continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);  in cached_dev_read_done_bh()
771 		continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);  in cached_dev_read_done_bh()
773 		continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);  in cached_dev_read_done_bh()
824 	cache_bio->bi_private	= &s->cl;  in cached_dev_cache_miss()
836 	closure_bio_submit(cache_bio, &s->cl, s->d);  in cached_dev_cache_miss()
843 	miss->bi_private	= &s->cl;  in cached_dev_cache_miss()
844 	closure_bio_submit(miss, &s->cl, s->d);  in cached_dev_cache_miss()
850 	struct closure *cl = &s->cl;  in cached_dev_read()  local
852 	closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in cached_dev_read()
853 	continue_at(cl, cached_dev_read_done_bh, NULL);  in cached_dev_read()
858 static void cached_dev_write_complete(struct closure *cl)  in cached_dev_write_complete()  argument
860 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_write_complete()
864 	cached_dev_bio_complete(cl);  in cached_dev_write_complete()
869 	struct closure *cl = &s->cl;  in cached_dev_write()  local
909 			closure_bio_submit(bio, cl, s->d);  in cached_dev_write()
922 			flush->bi_private = cl;  in cached_dev_write()
924 			closure_bio_submit(flush, cl, s->d);  in cached_dev_write()
929 		closure_bio_submit(bio, cl, s->d);  in cached_dev_write()
932 	closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_write()
933 	continue_at(cl, cached_dev_write_complete, NULL);  in cached_dev_write()
936 static void cached_dev_nodata(struct closure *cl)  in cached_dev_nodata()  argument
938 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_nodata()
942 		bch_journal_meta(s->iop.c, cl);  in cached_dev_nodata()
945 	closure_bio_submit(bio, cl, s->d);  in cached_dev_nodata()
947 	continue_at(cl, cached_dev_bio_complete, NULL);  in cached_dev_nodata()
973 			continue_at_nobarrier(&s->cl,  in cached_dev_make_request()
1054 static void flash_dev_nodata(struct closure *cl)  in flash_dev_nodata()  argument
1056 	struct search *s = container_of(cl, struct search, cl);  in flash_dev_nodata()
1059 		bch_journal_meta(s->iop.c, cl);  in flash_dev_nodata()
1061 	continue_at(cl, search_free, NULL);  in flash_dev_nodata()
1067 	struct closure *cl;  in flash_dev_make_request()  local
1074 	cl = &s->cl;  in flash_dev_make_request()
1084 		continue_at_nobarrier(&s->cl,  in flash_dev_make_request()
1096 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in flash_dev_make_request()
1098 		closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in flash_dev_make_request()
1101 	continue_at(cl, search_free, NULL);  in flash_dev_make_request()