254 			struct rbd_img_request	*img_request;  member
419 static int rbd_img_request_submit(struct rbd_img_request *img_request);
1401 		rbd_dev = obj_request->img_request->rbd_dev;  in obj_request_img_data_set()
1419 			rbd_dev = obj_request->img_request->rbd_dev;  in obj_request_done_set()
1464 	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;  in obj_request_overlaps_parent()
1486 static void rbd_img_request_get(struct rbd_img_request *img_request)  in rbd_img_request_get()  argument
1488 	dout("%s: img %p (was %d)\n", __func__, img_request,  in rbd_img_request_get()
1489 	     atomic_read(&img_request->kref.refcount));  in rbd_img_request_get()
1490 	kref_get(&img_request->kref);  in rbd_img_request_get()
1493 static bool img_request_child_test(struct rbd_img_request *img_request);
1496 static void rbd_img_request_put(struct rbd_img_request *img_request)  in rbd_img_request_put()  argument
1498 	rbd_assert(img_request != NULL);  in rbd_img_request_put()
1499 	dout("%s: img %p (was %d)\n", __func__, img_request,  in rbd_img_request_put()
1500 		atomic_read(&img_request->kref.refcount));  in rbd_img_request_put()
1501 	if (img_request_child_test(img_request))  in rbd_img_request_put()
1502 		kref_put(&img_request->kref, rbd_parent_request_destroy);  in rbd_img_request_put()
1504 		kref_put(&img_request->kref, rbd_img_request_destroy);  in rbd_img_request_put()
1507 static inline void rbd_img_obj_request_add(struct rbd_img_request *img_request,  in rbd_img_obj_request_add()  argument
1510 	rbd_assert(obj_request->img_request == NULL);  in rbd_img_obj_request_add()
1513 	obj_request->img_request = img_request;  in rbd_img_obj_request_add()
1514 	obj_request->which = img_request->obj_request_count;  in rbd_img_obj_request_add()
1518 	img_request->obj_request_count++;  in rbd_img_obj_request_add()
1519 	list_add_tail(&obj_request->links, &img_request->obj_requests);  in rbd_img_obj_request_add()
1520 	dout("%s: img %p obj %p w=%u\n", __func__, img_request, obj_request,  in rbd_img_obj_request_add()
1524 static inline void rbd_img_obj_request_del(struct rbd_img_request *img_request,  in rbd_img_obj_request_del()  argument
1529 	dout("%s: img %p obj %p w=%u\n", __func__, img_request, obj_request,  in rbd_img_obj_request_del()
1532 	rbd_assert(img_request->obj_request_count > 0);  in rbd_img_obj_request_del()
1533 	img_request->obj_request_count--;  in rbd_img_obj_request_del()
1534 	rbd_assert(obj_request->which == img_request->obj_request_count);  in rbd_img_obj_request_del()
1537 	rbd_assert(obj_request->img_request == img_request);  in rbd_img_obj_request_del()
1538 	obj_request->img_request = NULL;  in rbd_img_obj_request_del()
1606 static void rbd_img_request_complete(struct rbd_img_request *img_request)  in rbd_img_request_complete()  argument
1609 	dout("%s: img %p\n", __func__, img_request);  in rbd_img_request_complete()
1617 	if (!img_request->result) {  in rbd_img_request_complete()
1621 		for_each_obj_request(img_request, obj_request)  in rbd_img_request_complete()
1623 		img_request->xferred = xferred;  in rbd_img_request_complete()
1626 	if (img_request->callback)  in rbd_img_request_complete()
1627 		img_request->callback(img_request);  in rbd_img_request_complete()
1629 		rbd_img_request_put(img_request);  in rbd_img_request_complete()
1637 static void img_request_write_set(struct rbd_img_request *img_request)  in img_request_write_set()  argument
1639 	set_bit(IMG_REQ_WRITE, &img_request->flags);  in img_request_write_set()
1643 static bool img_request_write_test(struct rbd_img_request *img_request)  in img_request_write_test()  argument
1646 	return test_bit(IMG_REQ_WRITE, &img_request->flags) != 0;  in img_request_write_test()
1652 static void img_request_discard_set(struct rbd_img_request *img_request)  in img_request_discard_set()  argument
1654 	set_bit(IMG_REQ_DISCARD, &img_request->flags);  in img_request_discard_set()
1658 static bool img_request_discard_test(struct rbd_img_request *img_request)  in img_request_discard_test()  argument
1661 	return test_bit(IMG_REQ_DISCARD, &img_request->flags) != 0;  in img_request_discard_test()
1664 static void img_request_child_set(struct rbd_img_request *img_request)  in img_request_child_set()  argument
1666 	set_bit(IMG_REQ_CHILD, &img_request->flags);  in img_request_child_set()
1670 static void img_request_child_clear(struct rbd_img_request *img_request)  in img_request_child_clear()  argument
1672 	clear_bit(IMG_REQ_CHILD, &img_request->flags);  in img_request_child_clear()
1676 static bool img_request_child_test(struct rbd_img_request *img_request)  in img_request_child_test()  argument
1679 	return test_bit(IMG_REQ_CHILD, &img_request->flags) != 0;  in img_request_child_test()
1682 static void img_request_layered_set(struct rbd_img_request *img_request)  in img_request_layered_set()  argument
1684 	set_bit(IMG_REQ_LAYERED, &img_request->flags);  in img_request_layered_set()
1688 static void img_request_layered_clear(struct rbd_img_request *img_request)  in img_request_layered_clear()  argument
1690 	clear_bit(IMG_REQ_LAYERED, &img_request->flags);  in img_request_layered_clear()
1694 static bool img_request_layered_test(struct rbd_img_request *img_request)  in img_request_layered_test()  argument
1697 	return test_bit(IMG_REQ_LAYERED, &img_request->flags) != 0;  in img_request_layered_test()
1701 rbd_img_request_op_type(struct rbd_img_request *img_request)  in rbd_img_request_op_type()  argument
1703 	if (img_request_write_test(img_request))  in rbd_img_request_op_type()
1705 	else if (img_request_discard_test(img_request))  in rbd_img_request_op_type()
1718 		obj_request, obj_request->img_request, obj_request->result,  in rbd_img_obj_request_read_callback()
1763 	struct rbd_img_request *img_request = NULL;  in rbd_osd_read_callback()  local
1768 		img_request = obj_request->img_request;  in rbd_osd_read_callback()
1769 		layered = img_request && img_request_layered_test(img_request);  in rbd_osd_read_callback()
1770 		rbd_dev = img_request->rbd_dev;  in rbd_osd_read_callback()
1774 		obj_request, img_request, obj_request->result,  in rbd_osd_read_callback()
1779 	else if (img_request)  in rbd_osd_read_callback()
1841 		rbd_assert(obj_request->img_request);  in rbd_osd_req_callback()
1900 	struct rbd_img_request *img_request = obj_request->img_request;  in rbd_osd_req_format_read()  local
1906 	snap_id = img_request ? img_request->snap_id : CEPH_NOSNAP;  in rbd_osd_req_format_read()
1913 	struct rbd_img_request *img_request = obj_request->img_request;  in rbd_osd_req_format_write()  local
1920 	snapc = img_request ? img_request->snapc : NULL;  in rbd_osd_req_format_write()
1943 		struct rbd_img_request *img_request = obj_request->img_request;  in rbd_osd_req_create()  local
1945 			rbd_assert(img_request_write_test(img_request));  in rbd_osd_req_create()
1947 			rbd_assert(img_request_discard_test(img_request));  in rbd_osd_req_create()
1949 		snapc = img_request->snapc;  in rbd_osd_req_create()
1985 	struct rbd_img_request *img_request;  in rbd_osd_req_create_copyup()  local
1993 	img_request = obj_request->img_request;  in rbd_osd_req_create_copyup()
1994 	rbd_assert(img_request);  in rbd_osd_req_create_copyup()
1995 	rbd_assert(img_request_write_test(img_request) ||  in rbd_osd_req_create_copyup()
1996 			img_request_discard_test(img_request));  in rbd_osd_req_create_copyup()
1998 	if (img_request_discard_test(img_request))  in rbd_osd_req_create_copyup()
2003 	snapc = img_request->snapc;  in rbd_osd_req_create_copyup()
2004 	rbd_dev = img_request->rbd_dev;  in rbd_osd_req_create_copyup()
2074 	rbd_assert(obj_request->img_request == NULL);  in rbd_obj_request_destroy()
2173 	struct rbd_img_request *img_request;  in rbd_img_request_create()  local
2175 	img_request = kmem_cache_alloc(rbd_img_request_cache, GFP_NOIO);  in rbd_img_request_create()
2176 	if (!img_request)  in rbd_img_request_create()
2179 	img_request->rq = NULL;  in rbd_img_request_create()
2180 	img_request->rbd_dev = rbd_dev;  in rbd_img_request_create()
2181 	img_request->offset = offset;  in rbd_img_request_create()
2182 	img_request->length = length;  in rbd_img_request_create()
2183 	img_request->flags = 0;  in rbd_img_request_create()
2185 		img_request_discard_set(img_request);  in rbd_img_request_create()
2186 		img_request->snapc = snapc;  in rbd_img_request_create()
2188 		img_request_write_set(img_request);  in rbd_img_request_create()
2189 		img_request->snapc = snapc;  in rbd_img_request_create()
2191 		img_request->snap_id = rbd_dev->spec->snap_id;  in rbd_img_request_create()
2194 		img_request_layered_set(img_request);  in rbd_img_request_create()
2195 	spin_lock_init(&img_request->completion_lock);  in rbd_img_request_create()
2196 	img_request->next_completion = 0;  in rbd_img_request_create()
2197 	img_request->callback = NULL;  in rbd_img_request_create()
2198 	img_request->result = 0;  in rbd_img_request_create()
2199 	img_request->obj_request_count = 0;  in rbd_img_request_create()
2200 	INIT_LIST_HEAD(&img_request->obj_requests);  in rbd_img_request_create()
2201 	kref_init(&img_request->kref);  in rbd_img_request_create()
2204 		obj_op_name(op_type), offset, length, img_request);  in rbd_img_request_create()
2206 	return img_request;  in rbd_img_request_create()
2211 	struct rbd_img_request *img_request;  in rbd_img_request_destroy()  local
2215 	img_request = container_of(kref, struct rbd_img_request, kref);  in rbd_img_request_destroy()
2217 	dout("%s: img %p\n", __func__, img_request);  in rbd_img_request_destroy()
2219 	for_each_obj_request_safe(img_request, obj_request, next_obj_request)  in rbd_img_request_destroy()
2220 		rbd_img_obj_request_del(img_request, obj_request);  in rbd_img_request_destroy()
2221 	rbd_assert(img_request->obj_request_count == 0);  in rbd_img_request_destroy()
2223 	if (img_request_layered_test(img_request)) {  in rbd_img_request_destroy()
2224 		img_request_layered_clear(img_request);  in rbd_img_request_destroy()
2225 		rbd_dev_parent_put(img_request->rbd_dev);  in rbd_img_request_destroy()
2228 	if (img_request_write_test(img_request) ||  in rbd_img_request_destroy()
2229 		img_request_discard_test(img_request))  in rbd_img_request_destroy()
2230 		ceph_put_snap_context(img_request->snapc);  in rbd_img_request_destroy()
2232 	kmem_cache_free(rbd_img_request_cache, img_request);  in rbd_img_request_destroy()
2242 	rbd_assert(obj_request->img_request);  in rbd_parent_request_create()
2243 	rbd_dev = obj_request->img_request->rbd_dev;  in rbd_parent_request_create()
2274 	struct rbd_img_request *img_request;  in rbd_img_obj_end_request()  local
2280 	img_request = obj_request->img_request;  in rbd_img_obj_end_request()
2286 		struct rbd_device *rbd_dev = img_request->rbd_dev;  in rbd_img_obj_end_request()
2289 		if (img_request_discard_test(img_request))  in rbd_img_obj_end_request()
2291 		else if (img_request_write_test(img_request))  in rbd_img_obj_end_request()
2301 		if (!img_request->result)  in rbd_img_obj_end_request()
2302 			img_request->result = result;  in rbd_img_obj_end_request()
2317 	if (img_request_child_test(img_request)) {  in rbd_img_obj_end_request()
2318 		rbd_assert(img_request->obj_request != NULL);  in rbd_img_obj_end_request()
2319 		more = obj_request->which < img_request->obj_request_count - 1;  in rbd_img_obj_end_request()
2321 		rbd_assert(img_request->rq != NULL);  in rbd_img_obj_end_request()
2323 		more = blk_update_request(img_request->rq, result, xferred);  in rbd_img_obj_end_request()
2325 			__blk_mq_end_request(img_request->rq, result);  in rbd_img_obj_end_request()
2333 	struct rbd_img_request *img_request;  in rbd_img_obj_callback()  local
2338 	img_request = obj_request->img_request;  in rbd_img_obj_callback()
2340 	dout("%s: img %p obj %p\n", __func__, img_request, obj_request);  in rbd_img_obj_callback()
2341 	rbd_assert(img_request != NULL);  in rbd_img_obj_callback()
2342 	rbd_assert(img_request->obj_request_count > 0);  in rbd_img_obj_callback()
2344 	rbd_assert(which < img_request->obj_request_count);  in rbd_img_obj_callback()
2346 	spin_lock_irq(&img_request->completion_lock);  in rbd_img_obj_callback()
2347 	if (which != img_request->next_completion)  in rbd_img_obj_callback()
2350 	for_each_obj_request_from(img_request, obj_request) {  in rbd_img_obj_callback()
2352 		rbd_assert(which < img_request->obj_request_count);  in rbd_img_obj_callback()
2360 	rbd_assert(more ^ (which == img_request->obj_request_count));  in rbd_img_obj_callback()
2361 	img_request->next_completion = which;  in rbd_img_obj_callback()
2363 	spin_unlock_irq(&img_request->completion_lock);  in rbd_img_obj_callback()
2364 	rbd_img_request_put(img_request);  in rbd_img_obj_callback()
2367 		rbd_img_request_complete(img_request);  in rbd_img_obj_callback()
2380 	struct rbd_img_request *img_request = obj_request->img_request;  in rbd_img_obj_request_fill()  local
2381 	struct rbd_device *rbd_dev = img_request->rbd_dev;  in rbd_img_obj_request_fill()
2390 		    (!img_request_layered_test(img_request) ||  in rbd_img_obj_request_fill()
2446 static int rbd_img_request_fill(struct rbd_img_request *img_request,  in rbd_img_request_fill()  argument
2450 	struct rbd_device *rbd_dev = img_request->rbd_dev;  in rbd_img_request_fill()
2460 	dout("%s: img %p type %d data_desc %p\n", __func__, img_request,  in rbd_img_request_fill()
2463 	img_offset = img_request->offset;  in rbd_img_request_fill()
2464 	resid = img_request->length;  in rbd_img_request_fill()
2466 	op_type = rbd_img_request_op_type(img_request);  in rbd_img_request_fill()
2498 		rbd_img_obj_request_add(img_request, obj_request);  in rbd_img_request_fill()
2535 		rbd_img_request_get(img_request);  in rbd_img_request_fill()
2544 	for_each_obj_request_safe(img_request, obj_request, next_obj_request)  in rbd_img_request_fill()
2545 		rbd_img_obj_request_del(img_request, obj_request);  in rbd_img_request_fill()
2553 	struct rbd_img_request *img_request;  in rbd_osd_copyup_callback()  local
2563 	img_request = obj_request->img_request;  in rbd_osd_copyup_callback()
2564 	rbd_assert(img_request);  in rbd_osd_copyup_callback()
2566 	rbd_dev = img_request->rbd_dev;  in rbd_osd_copyup_callback()
2590 rbd_img_obj_parent_read_full_callback(struct rbd_img_request *img_request)  in rbd_img_obj_parent_read_full_callback()  argument
2602 	rbd_assert(img_request_child_test(img_request));  in rbd_img_obj_parent_read_full_callback()
2606 	pages = img_request->copyup_pages;  in rbd_img_obj_parent_read_full_callback()
2608 	img_request->copyup_pages = NULL;  in rbd_img_obj_parent_read_full_callback()
2609 	page_count = img_request->copyup_page_count;  in rbd_img_obj_parent_read_full_callback()
2611 	img_request->copyup_page_count = 0;  in rbd_img_obj_parent_read_full_callback()
2613 	orig_request = img_request->obj_request;  in rbd_img_obj_parent_read_full_callback()
2616 	img_result = img_request->result;  in rbd_img_obj_parent_read_full_callback()
2617 	parent_length = img_request->length;  in rbd_img_obj_parent_read_full_callback()
2618 	rbd_assert(parent_length == img_request->xferred);  in rbd_img_obj_parent_read_full_callback()
2619 	rbd_img_request_put(img_request);  in rbd_img_obj_parent_read_full_callback()
2621 	rbd_assert(orig_request->img_request);  in rbd_img_obj_parent_read_full_callback()
2622 	rbd_dev = orig_request->img_request->rbd_dev;  in rbd_img_obj_parent_read_full_callback()
2666 	op_type = rbd_img_request_op_type(orig_request->img_request);  in rbd_img_obj_parent_read_full_callback()
2700 	struct rbd_img_request *img_request = NULL;  in rbd_img_obj_parent_read_full()  local
2712 	img_request = obj_request->img_request;  in rbd_img_obj_parent_read_full()
2713 	rbd_assert(img_request != NULL);  in rbd_img_obj_parent_read_full()
2714 	rbd_dev = img_request->rbd_dev;  in rbd_img_obj_parent_read_full()
2796 	rbd_assert(orig_request->img_request);  in rbd_img_obj_exists_callback()
2811 	rbd_dev = orig_request->img_request->rbd_dev;  in rbd_img_obj_exists_callback()
2881 	rbd_assert(obj_request->img_request);  in rbd_img_obj_exists_submit()
2882 	rbd_dev = obj_request->img_request->rbd_dev;  in rbd_img_obj_exists_submit()
2905 	struct rbd_img_request *img_request;  in img_obj_request_simple()  local
2910 	img_request = obj_request->img_request;  in img_obj_request_simple()
2911 	rbd_assert(img_request);  in img_obj_request_simple()
2912 	rbd_dev = img_request->rbd_dev;  in img_obj_request_simple()
2915 	if (!img_request_write_test(img_request) &&  in img_obj_request_simple()
2916 	    !img_request_discard_test(img_request))  in img_obj_request_simple()
2920 	if (!img_request_layered_test(img_request))  in img_obj_request_simple()
2955 		rbd_dev = obj_request->img_request->rbd_dev;  in rbd_img_obj_request_submit()
2975 static int rbd_img_request_submit(struct rbd_img_request *img_request)  in rbd_img_request_submit()  argument
2980 	dout("%s: img %p\n", __func__, img_request);  in rbd_img_request_submit()
2981 	for_each_obj_request_safe(img_request, obj_request, next_obj_request) {  in rbd_img_request_submit()
2992 static void rbd_img_parent_read_callback(struct rbd_img_request *img_request)  in rbd_img_parent_read_callback()  argument
3000 	rbd_assert(img_request_child_test(img_request));  in rbd_img_parent_read_callback()
3004 	obj_request = img_request->obj_request;  in rbd_img_parent_read_callback()
3005 	img_xferred = img_request->xferred;  in rbd_img_parent_read_callback()
3006 	img_result = img_request->result;  in rbd_img_parent_read_callback()
3007 	rbd_img_request_put(img_request);  in rbd_img_parent_read_callback()
3015 	rbd_assert(obj_request->img_request);  in rbd_img_parent_read_callback()
3016 	rbd_dev = obj_request->img_request->rbd_dev;  in rbd_img_parent_read_callback()
3057 	struct rbd_img_request *img_request;  in rbd_img_parent_read()  local
3061 	rbd_assert(obj_request->img_request != NULL);  in rbd_img_parent_read()
3066 	img_request = rbd_parent_request_create(obj_request,  in rbd_img_parent_read()
3070 	if (!img_request)  in rbd_img_parent_read()
3074 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_BIO,  in rbd_img_parent_read()
3077 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_PAGES,  in rbd_img_parent_read()
3082 	img_request->callback = rbd_img_parent_read_callback;  in rbd_img_parent_read()
3083 	result = rbd_img_request_submit(img_request);  in rbd_img_parent_read()
3089 	if (img_request)  in rbd_img_parent_read()
3090 		rbd_img_request_put(img_request);  in rbd_img_parent_read()
3361 	struct rbd_img_request *img_request;  in rbd_queue_workfn()  local
3438 	img_request = rbd_img_request_create(rbd_dev, offset, length, op_type,  in rbd_queue_workfn()
3440 	if (!img_request) {  in rbd_queue_workfn()
3444 	img_request->rq = rq;  in rbd_queue_workfn()
3448 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_NODATA,  in rbd_queue_workfn()
3451 		result = rbd_img_request_fill(img_request, OBJ_REQUEST_BIO,  in rbd_queue_workfn()
3456 	result = rbd_img_request_submit(img_request);  in rbd_queue_workfn()
3463 	rbd_img_request_put(img_request);  in rbd_queue_workfn()