254 			struct rbd_img_request	*img_request;  member
418 static int rbd_img_request_submit(struct rbd_img_request *img_request);
1399 		rbd_dev = obj_request->img_request->rbd_dev;  in obj_request_img_data_set()
1417 			rbd_dev = obj_request->img_request->rbd_dev;  in obj_request_done_set()
1462 	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;  in obj_request_overlaps_parent()
1484 static void rbd_img_request_get(struct rbd_img_request *img_request)  in rbd_img_request_get()  argument
1486 	dout("%s: img %p (was %d)\n", __func__, img_request,  in rbd_img_request_get()
1487 	     atomic_read(&img_request->kref.refcount));  in rbd_img_request_get()
1488 	kref_get(&img_request->kref);  in rbd_img_request_get()
1491 static bool img_request_child_test(struct rbd_img_request *img_request);
1494 static void rbd_img_request_put(struct rbd_img_request *img_request)  in rbd_img_request_put()  argument
1496 	rbd_assert(img_request != NULL);  in rbd_img_request_put()
1497 	dout("%s: img %p (was %d)\n", __func__, img_request,  in rbd_img_request_put()
1498 		atomic_read(&img_request->kref.refcount));  in rbd_img_request_put()
1499 	if (img_request_child_test(img_request))  in rbd_img_request_put()
1500 		kref_put(&img_request->kref, rbd_parent_request_destroy);  in rbd_img_request_put()
1502 		kref_put(&img_request->kref, rbd_img_request_destroy);  in rbd_img_request_put()
1505 static inline void rbd_img_obj_request_add(struct rbd_img_request *img_request,  in rbd_img_obj_request_add()  argument
1508 	rbd_assert(obj_request->img_request == NULL);  in rbd_img_obj_request_add()
1511 	obj_request->img_request = img_request;  in rbd_img_obj_request_add()
1512 	obj_request->which = img_request->obj_request_count;  in rbd_img_obj_request_add()
1516 	img_request->obj_request_count++;  in rbd_img_obj_request_add()
1517 	list_add_tail(&obj_request->links, &img_request->obj_requests);  in rbd_img_obj_request_add()
1518 	dout("%s: img %p obj %p w=%u\n", __func__, img_request, obj_request,  in rbd_img_obj_request_add()
1522 static inline void rbd_img_obj_request_del(struct rbd_img_request *img_request,  in rbd_img_obj_request_del()  argument
1527 	dout("%s: img %p obj %p w=%u\n", __func__, img_request, obj_request,  in rbd_img_obj_request_del()
1530 	rbd_assert(img_request->obj_request_count > 0);  in rbd_img_obj_request_del()
1531 	img_request->obj_request_count--;  in rbd_img_obj_request_del()
1532 	rbd_assert(obj_request->which == img_request->obj_request_count);  in rbd_img_obj_request_del()
1535 	rbd_assert(obj_request->img_request == img_request);  in rbd_img_obj_request_del()
1536 	obj_request->img_request = NULL;  in rbd_img_obj_request_del()
1587 static void rbd_img_request_complete(struct rbd_img_request *img_request)  in rbd_img_request_complete()  argument
1590 	dout("%s: img %p\n", __func__, img_request);  in rbd_img_request_complete()
1598 	if (!img_request->result) {  in rbd_img_request_complete()
1602 		for_each_obj_request(img_request, obj_request)  in rbd_img_request_complete()
1604 		img_request->xferred = xferred;  in rbd_img_request_complete()
1607 	if (img_request->callback)  in rbd_img_request_complete()
1608 		img_request->callback(img_request);  in rbd_img_request_complete()
1610 		rbd_img_request_put(img_request);  in rbd_img_request_complete()
1618 static void img_request_write_set(struct rbd_img_request *img_request)  in img_request_write_set()  argument
1620 	set_bit(IMG_REQ_WRITE, &img_request->flags);  in img_request_write_set()
1624 static bool img_request_write_test(struct rbd_img_request *img_request)  in img_request_write_test()  argument
1627 	return test_bit(IMG_REQ_WRITE, &img_request->flags) != 0;  in img_request_write_test()
1633 static void img_request_discard_set(struct rbd_img_request *img_request)  in img_request_discard_set()  argument
1635 	set_bit(IMG_REQ_DISCARD, &img_request->flags);  in img_request_discard_set()
1639 static bool img_request_discard_test(struct rbd_img_request *img_request)  in img_request_discard_test()  argument
1642 	return test_bit(IMG_REQ_DISCARD, &img_request->flags) != 0;  in img_request_discard_test()
1645 static void img_request_child_set(struct rbd_img_request *img_request)  in img_request_child_set()  argument
1647 	set_bit(IMG_REQ_CHILD, &img_request->flags);  in img_request_child_set()
1651 static void img_request_child_clear(struct rbd_img_request *img_request)  in img_request_child_clear()  argument
1653 	clear_bit(IMG_REQ_CHILD, &img_request->flags);  in img_request_child_clear()
1657 static bool img_request_child_test(struct rbd_img_request *img_request)  in img_request_child_test()  argument
1660 	return test_bit(IMG_REQ_CHILD, &img_request->flags) != 0;  in img_request_child_test()
1663 static void img_request_layered_set(struct rbd_img_request *img_request)  in img_request_layered_set()  argument
1665 	set_bit(IMG_REQ_LAYERED, &img_request->flags);  in img_request_layered_set()
1669 static void img_request_layered_clear(struct rbd_img_request *img_request)  in img_request_layered_clear()  argument
1671 	clear_bit(IMG_REQ_LAYERED, &img_request->flags);  in img_request_layered_clear()
1675 static bool img_request_layered_test(struct rbd_img_request *img_request)  in img_request_layered_test()  argument
1678 	return test_bit(IMG_REQ_LAYERED, &img_request->flags) != 0;  in img_request_layered_test()
1682 rbd_img_request_op_type(struct rbd_img_request *img_request)  in rbd_img_request_op_type()  argument
1684 	if (img_request_write_test(img_request))  in rbd_img_request_op_type()
1686 	else if (img_request_discard_test(img_request))  in rbd_img_request_op_type()
1699 		obj_request, obj_request->img_request, obj_request->result,  in rbd_img_obj_request_read_callback()
1744 	struct rbd_img_request *img_request = NULL;  in rbd_osd_read_callback()  local
1749 		img_request = obj_request->img_request;  in rbd_osd_read_callback()
1750 		layered = img_request && img_request_layered_test(img_request);  in rbd_osd_read_callback()
1751 		rbd_dev = img_request->rbd_dev;  in rbd_osd_read_callback()
1755 		obj_request, img_request, obj_request->result,  in rbd_osd_read_callback()
1760 	else if (img_request)  in rbd_osd_read_callback()
1822 		rbd_assert(obj_request->img_request);  in rbd_osd_req_callback()
1879 	struct rbd_img_request *img_request = obj_request->img_request;  in rbd_osd_req_format_read()  local
1885 	snap_id = img_request ? img_request->snap_id : CEPH_NOSNAP;  in rbd_osd_req_format_read()
1892 	struct rbd_img_request *img_request = obj_request->img_request;  in rbd_osd_req_format_write()  local
1899 	snapc = img_request ? img_request->snapc : NULL;  in rbd_osd_req_format_write()
1922 		struct rbd_img_request *img_request = obj_request->img_request;  in rbd_osd_req_create()  local
1924 			rbd_assert(img_request_write_test(img_request));  in rbd_osd_req_create()
1926 			rbd_assert(img_request_discard_test(img_request));  in rbd_osd_req_create()
1928 		snapc = img_request->snapc;  in rbd_osd_req_create()
1964 	struct rbd_img_request *img_request;  in rbd_osd_req_create_copyup()  local
1972 	img_request = obj_request->img_request;  in rbd_osd_req_create_copyup()
1973 	rbd_assert(img_request);  in rbd_osd_req_create_copyup()
1974 	rbd_assert(img_request_write_test(img_request) ||  in rbd_osd_req_create_copyup()
1975 			img_request_discard_test(img_request));  in rbd_osd_req_create_copyup()
1977 	if (img_request_discard_test(img_request))  in rbd_osd_req_create_copyup()
1982 	snapc = img_request->snapc;  in rbd_osd_req_create_copyup()
1983 	rbd_dev = img_request->rbd_dev;  in rbd_osd_req_create_copyup()
2053 	rbd_assert(obj_request->img_request == NULL);  in rbd_obj_request_destroy()
2152 	struct rbd_img_request *img_request;  in rbd_img_request_create()  local
2154 	img_request = kmem_cache_alloc(rbd_img_request_cache, GFP_NOIO);  in rbd_img_request_create()
2155 	if (!img_request)  in rbd_img_request_create()
2158 	img_request->rq = NULL;  in rbd_img_request_create()
2159 	img_request->rbd_dev = rbd_dev;  in rbd_img_request_create()
2160 	img_request->offset = offset;  in rbd_img_request_create()
2161 	img_request->length = length;  in rbd_img_request_create()
2162 	img_request->flags = 0;  in rbd_img_request_create()
2164 		img_request_discard_set(img_request);  in rbd_img_request_create()
2165 		img_request->snapc = snapc;  in rbd_img_request_create()
2167 		img_request_write_set(img_request);  in rbd_img_request_create()
2168 		img_request->snapc = snapc;  in rbd_img_request_create()
2170 		img_request->snap_id = rbd_dev->spec->snap_id;  in rbd_img_request_create()
2173 		img_request_layered_set(img_request);  in rbd_img_request_create()
2174 	spin_lock_init(&img_request->completion_lock);  in rbd_img_request_create()
2175 	img_request->next_completion = 0;  in rbd_img_request_create()
2176 	img_request->callback = NULL;  in rbd_img_request_create()
2177 	img_request->result = 0;  in rbd_img_request_create()
2178 	img_request->obj_request_count = 0;  in rbd_img_request_create()
2179 	INIT_LIST_HEAD(&img_request->obj_requests);  in rbd_img_request_create()
2180 	kref_init(&img_request->kref);  in rbd_img_request_create()
2183 		obj_op_name(op_type), offset, length, img_request);  in rbd_img_request_create()
2185 	return img_request;  in rbd_img_request_create()
2190 	struct rbd_img_request *img_request;  in rbd_img_request_destroy()  local
2194 	img_request = container_of(kref, struct rbd_img_request, kref);  in rbd_img_request_destroy()
2196 	dout("%s: img %p\n", __func__, img_request);  in rbd_img_request_destroy()
2198 	for_each_obj_request_safe(img_request, obj_request, next_obj_request)  in rbd_img_request_destroy()
2199 		rbd_img_obj_request_del(img_request, obj_request);  in rbd_img_request_destroy()
2200 	rbd_assert(img_request->obj_request_count == 0);  in rbd_img_request_destroy()
2202 	if (img_request_layered_test(img_request)) {  in rbd_img_request_destroy()
2203 		img_request_layered_clear(img_request);  in rbd_img_request_destroy()
2204 		rbd_dev_parent_put(img_request->rbd_dev);  in rbd_img_request_destroy()
2207 	if (img_request_write_test(img_request) ||  in rbd_img_request_destroy()
2208 		img_request_discard_test(img_request))  in rbd_img_request_destroy()
2209 		ceph_put_snap_context(img_request->snapc);  in rbd_img_request_destroy()
2211 	kmem_cache_free(rbd_img_request_cache, img_request);  in rbd_img_request_destroy()
2221 	rbd_assert(obj_request->img_request);  in rbd_parent_request_create()
2222 	rbd_dev = obj_request->img_request->rbd_dev;  in rbd_parent_request_create()
2253 	struct rbd_img_request *img_request;  in rbd_img_obj_end_request()  local
2259 	img_request = obj_request->img_request;  in rbd_img_obj_end_request()
2265 		struct rbd_device *rbd_dev = img_request->rbd_dev;  in rbd_img_obj_end_request()
2268 		if (img_request_discard_test(img_request))  in rbd_img_obj_end_request()
2270 		else if (img_request_write_test(img_request))  in rbd_img_obj_end_request()
2280 		if (!img_request->result)  in rbd_img_obj_end_request()
2281 			img_request->result = result;  in rbd_img_obj_end_request()
2296 	if (img_request_child_test(img_request)) {  in rbd_img_obj_end_request()
2297 		rbd_assert(img_request->obj_request != NULL);  in rbd_img_obj_end_request()
2298 		more = obj_request->which < img_request->obj_request_count - 1;  in rbd_img_obj_end_request()
2300 		rbd_assert(img_request->rq != NULL);  in rbd_img_obj_end_request()
2302 		more = blk_update_request(img_request->rq, result, xferred);  in rbd_img_obj_end_request()
2304 			__blk_mq_end_request(img_request->rq, result);  in rbd_img_obj_end_request()
2312 	struct rbd_img_request *img_request;  in rbd_img_obj_callback()  local
2317 	img_request = obj_request->img_request;  in rbd_img_obj_callback()
2319 	dout("%s: img %p obj %p\n", __func__, img_request, obj_request);  in rbd_img_obj_callback()
2320 	rbd_assert(img_request != NULL);  in rbd_img_obj_callback()
2321 	rbd_assert(img_request->obj_request_count > 0);  in rbd_img_obj_callback()
2323 	rbd_assert(which < img_request->obj_request_count);  in rbd_img_obj_callback()
2325 	spin_lock_irq(&img_request->completion_lock);  in rbd_img_obj_callback()
2326 	if (which != img_request->next_completion)  in rbd_img_obj_callback()
2329 	for_each_obj_request_from(img_request, obj_request) {  in rbd_img_obj_callback()
2331 		rbd_assert(which < img_request->obj_request_count);  in rbd_img_obj_callback()
2339 	rbd_assert(more ^ (which == img_request->obj_request_count));  in rbd_img_obj_callback()
2340 	img_request->next_completion = which;  in rbd_img_obj_callback()
2342 	spin_unlock_irq(&img_request->completion_lock);  in rbd_img_obj_callback()
2343 	rbd_img_request_put(img_request);  in rbd_img_obj_callback()
2346 		rbd_img_request_complete(img_request);  in rbd_img_obj_callback()
2359 	struct rbd_img_request *img_request = obj_request->img_request;  in rbd_img_obj_request_fill()  local
2360 	struct rbd_device *rbd_dev = img_request->rbd_dev;  in rbd_img_obj_request_fill()
2369 		    (!img_request_layered_test(img_request) ||  in rbd_img_obj_request_fill()
2422 static int rbd_img_request_fill(struct rbd_img_request *img_request,  in rbd_img_request_fill()  argument
2426 	struct rbd_device *rbd_dev = img_request->rbd_dev;  in rbd_img_request_fill()
2436 	dout("%s: img %p type %d data_desc %p\n", __func__, img_request,  in rbd_img_request_fill()
2439 	img_offset = img_request->offset;  in rbd_img_request_fill()
2440 	resid = img_request->length;  in rbd_img_request_fill()
2442 	op_type = rbd_img_request_op_type(img_request);  in rbd_img_request_fill()
2474 		rbd_img_obj_request_add(img_request, obj_request);  in rbd_img_request_fill()
2511 		rbd_img_request_get(img_request);  in rbd_img_request_fill()
2520 	for_each_obj_request_safe(img_request, obj_request, next_obj_request)  in rbd_img_request_fill()
2521 		rbd_img_obj_request_del(img_request, obj_request);  in rbd_img_request_fill()
2529 	struct rbd_img_request *img_request;  in rbd_osd_copyup_callback()  local
2539 	img_request = obj_request->img_request;  in rbd_osd_copyup_callback()
2540 	rbd_assert(img_request);  in rbd_osd_copyup_callback()
2542 	rbd_dev = img_request->rbd_dev;  in rbd_osd_copyup_callback()
2566 rbd_img_obj_parent_read_full_callback(struct rbd_img_request *img_request)  in rbd_img_obj_parent_read_full_callback()  argument
2578 	rbd_assert(img_request_child_test(img_request));  in rbd_img_obj_parent_read_full_callback()
2582 	pages = img_request->copyup_pages;  in rbd_img_obj_parent_read_full_callback()
2584 	img_request->copyup_pages = NULL;  in rbd_img_obj_parent_read_full_callback()
2585 	page_count = img_request->copyup_page_count;  in rbd_img_obj_parent_read_full_callback()
2587 	img_request->copyup_page_count = 0;  in rbd_img_obj_parent_read_full_callback()
2589 	orig_request = img_request->obj_request;  in rbd_img_obj_parent_read_full_callback()
2592 	img_result = img_request->result;  in rbd_img_obj_parent_read_full_callback()
2593 	parent_length = img_request->length;  in rbd_img_obj_parent_read_full_callback()
2594 	rbd_assert(parent_length == img_request->xferred);  in rbd_img_obj_parent_read_full_callback()
2595 	rbd_img_request_put(img_request);  in rbd_img_obj_parent_read_full_callback()
2597 	rbd_assert(orig_request->img_request);  in rbd_img_obj_parent_read_full_callback()
2598 	rbd_dev = orig_request->img_request->rbd_dev;  in rbd_img_obj_parent_read_full_callback()
2642 	op_type = rbd_img_request_op_type(orig_request->img_request);  in rbd_img_obj_parent_read_full_callback()
2676 	struct rbd_img_request *img_request = NULL;  in rbd_img_obj_parent_read_full()  local
2688 	img_request = obj_request->img_request;  in rbd_img_obj_parent_read_full()
2689 	rbd_assert(img_request != NULL);  in rbd_img_obj_parent_read_full()
2690 	rbd_dev = img_request->rbd_dev;  in rbd_img_obj_parent_read_full()
2772 	rbd_assert(orig_request->img_request);  in rbd_img_obj_exists_callback()
2787 	rbd_dev = orig_request->img_request->rbd_dev;  in rbd_img_obj_exists_callback()
2857 	rbd_assert(obj_request->img_request);  in rbd_img_obj_exists_submit()
2858 	rbd_dev = obj_request->img_request->rbd_dev;  in rbd_img_obj_exists_submit()
2881 	struct rbd_img_request *img_request;  in img_obj_request_simple()  local
2886 	img_request = obj_request->img_request;  in img_obj_request_simple()
2887 	rbd_assert(img_request);  in img_obj_request_simple()
2888 	rbd_dev = img_request->rbd_dev;  in img_obj_request_simple()
2891 	if (!img_request_write_test(img_request) &&  in img_obj_request_simple()
2892 	    !img_request_discard_test(img_request))  in img_obj_request_simple()
2896 	if (!img_request_layered_test(img_request))  in img_obj_request_simple()
2931 		rbd_dev = obj_request->img_request->rbd_dev;  in rbd_img_obj_request_submit()
2951 static int rbd_img_request_submit(struct rbd_img_request *img_request)  in rbd_img_request_submit()  argument
2956 	dout("%s: img %p\n", __func__, img_request);  in rbd_img_request_submit()
2957 	for_each_obj_request_safe(img_request, obj_request, next_obj_request) {  in rbd_img_request_submit()
2968 static void rbd_img_parent_read_callback(struct rbd_img_request *img_request)  in rbd_img_parent_read_callback()  argument
2976 	rbd_assert(img_request_child_test(img_request));  in rbd_img_parent_read_callback()
2980 	obj_request = img_request->obj_request;  in rbd_img_parent_read_callback()
2981 	img_xferred = img_request->xferred;  in rbd_img_parent_read_callback()
2982 	img_result = img_request->result;  in rbd_img_parent_read_callback()
2983 	rbd_img_request_put(img_request);  in rbd_img_parent_read_callback()
2991 	rbd_assert(obj_request->img_request);  in rbd_img_parent_read_callback()
2992 	rbd_dev = obj_request->img_request->rbd_dev;  in rbd_img_parent_read_callback()
3033 	struct rbd_img_request *img_request;  in rbd_img_parent_read()  local
3037 	rbd_assert(obj_request->img_request != NULL);  in rbd_img_parent_read()
3042 	img_request = rbd_parent_request_create(obj_request,  in rbd_img_parent_read()
3046 	if (!img_request)  in rbd_img_parent_read()
3050 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_BIO,  in rbd_img_parent_read()
3053 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_PAGES,  in rbd_img_parent_read()
3058 	img_request->callback = rbd_img_parent_read_callback;  in rbd_img_parent_read()
3059 	result = rbd_img_request_submit(img_request);  in rbd_img_parent_read()
3065 	if (img_request)  in rbd_img_parent_read()
3066 		rbd_img_request_put(img_request);  in rbd_img_parent_read()
3336 	struct rbd_img_request *img_request;  in rbd_queue_workfn()  local
3413 	img_request = rbd_img_request_create(rbd_dev, offset, length, op_type,  in rbd_queue_workfn()
3415 	if (!img_request) {  in rbd_queue_workfn()
3419 	img_request->rq = rq;  in rbd_queue_workfn()
3423 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_NODATA,  in rbd_queue_workfn()
3426 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_BIO,  in rbd_queue_workfn()
3431 	result = rbd_img_request_submit(img_request);  in rbd_queue_workfn()
3438 	rbd_img_request_put(img_request);  in rbd_queue_workfn()