59 #define RQ_CFQG(rq)		(struct cfq_group *) ((rq)->elv.priv[1])
68 #define rb_entry_cfqg(node)	rb_entry((node), struct cfq_group, rb_node)
148 	struct cfq_group *cfqg;
216 struct cfq_group {  struct
313 	struct cfq_group *root_group;  argument
321 	struct cfq_group *serving_group;
389 static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd);
391 static struct cfq_rb_root *st_for(struct cfq_group *cfqg,  in st_for()
449 static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)  in pd_to_cfqg()
451 	return pd ? container_of(pd, struct cfq_group, pd) : NULL;  in pd_to_cfqg()
454 static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)  in cfqg_to_blkg()
503 static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,  in cfqg_stats_set_start_group_wait_time()
504 						 struct cfq_group *curr_cfqg)  in cfqg_stats_set_start_group_wait_time()
531 static void cfqg_stats_update_dequeue(struct cfq_group *cfqg)  in cfqg_stats_update_dequeue()
536 static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)  in cfqg_stats_set_start_empty_time()
555 static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)  in cfqg_stats_update_idle_time()
569 static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)  in cfqg_stats_set_start_idle_time()
579 static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)  in cfqg_stats_update_avg_queue_size()
591 static inline void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg, struct cfq_group *c…  in cfqg_stats_set_start_group_wait_time()
593 static inline void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }  in cfqg_stats_update_dequeue()
594 static inline void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }  in cfqg_stats_set_start_empty_time()
595 static inline void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }  in cfqg_stats_update_idle_time()
596 static inline void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }  in cfqg_stats_set_start_idle_time()
597 static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }  in cfqg_stats_update_avg_queue_size()
605 static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)  in blkg_to_cfqg()
610 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)  in cfqg_parent()
617 static inline void cfqg_get(struct cfq_group *cfqg)  in cfqg_get()
622 static inline void cfqg_put(struct cfq_group *cfqg)  in cfqg_put()
644 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,  in cfqg_stats_update_io_add()
645 					    struct cfq_group *curr_cfqg, int rw)  in cfqg_stats_update_io_add()
652 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,  in cfqg_stats_update_timeslice_used()
661 static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw)  in cfqg_stats_update_io_remove()
666 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw)  in cfqg_stats_update_io_merged()
671 static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,  in cfqg_stats_update_dispatch()
679 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,  in cfqg_stats_update_completion()
739 static void cfqg_stats_xfer_dead(struct cfq_group *cfqg)  in cfqg_stats_xfer_dead()
741 	struct cfq_group *parent = cfqg_parent(cfqg);  in cfqg_stats_xfer_dead()
756 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg) { return NULL; }  in cfqg_parent()
757 static inline void cfqg_get(struct cfq_group *cfqg) { }  in cfqg_get()
758 static inline void cfqg_put(struct cfq_group *cfqg) { }  in cfqg_put()
767 static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,  in cfqg_stats_update_io_add()
768 			struct cfq_group *curr_cfqg, int rw) { }  in cfqg_stats_update_io_add()
769 static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,  in cfqg_stats_update_timeslice_used()
771 static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { }  in cfqg_stats_update_io_remove()
772 static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { }  in cfqg_stats_update_io_merged()
773 static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,  in cfqg_stats_update_dispatch()
775 static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,  in cfqg_stats_update_completion()
842 					struct cfq_group *cfqg)  in cfq_group_busy_queues_wl()
853 					struct cfq_group *cfqg)  in cfqg_busy_async_queues()
979 	struct cfq_group *cfqg;  in update_min_vdisktime()
995 					struct cfq_group *cfqg, bool rt)  in cfq_group_get_avg_queues()
1010 cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_slice()
1179 static struct cfq_group *cfq_rb_first_group(struct cfq_rb_root *root)  in cfq_rb_first_group()
1242 cfqg_key(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfqg_key()
1248 __cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)  in __cfq_group_service_tree_add()
1252 	struct cfq_group *__cfqg;  in __cfq_group_service_tree_add()
1279 cfq_update_group_weight(struct cfq_group *cfqg)  in cfq_update_group_weight()
1288 cfq_update_group_leaf_weight(struct cfq_group *cfqg)  in cfq_update_group_leaf_weight()
1299 cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfq_group_service_tree_add()
1302 	struct cfq_group *pos = cfqg;  in cfq_group_service_tree_add()
1303 	struct cfq_group *parent;  in cfq_group_service_tree_add()
1350 cfq_group_notify_queue_add(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_notify_queue_add()
1353 	struct cfq_group *__cfqg;  in cfq_group_notify_queue_add()
1375 cfq_group_service_tree_del(struct cfq_rb_root *st, struct cfq_group *cfqg)  in cfq_group_service_tree_del()
1377 	struct cfq_group *pos = cfqg;  in cfq_group_service_tree_del()
1388 		struct cfq_group *parent = cfqg_parent(pos);  in cfq_group_service_tree_del()
1408 cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg)  in cfq_group_notify_queue_del()
1457 static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,  in cfq_group_served()
1511 static void cfq_init_cfqg_base(struct cfq_group *cfqg)  in cfq_init_cfqg_base()
1549 	struct cfq_group *cfqg = blkg_to_cfqg(blkg);  in cfq_pd_init()
1570 static const int dead_stats_off_delta = offsetof(struct cfq_group, dead_stats) -
1571 					offsetof(struct cfq_group, stats);
1597 	struct cfq_group *cfqg = blkg_to_cfqg(blkg);  in cfq_pd_reset_stats()
1607 static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd,  in cfq_lookup_create_cfqg()
1611 	struct cfq_group *cfqg = NULL;  in cfq_lookup_create_cfqg()
1627 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)  in cfq_link_cfqq_cfqg()
1641 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_weight_device()
1659 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_leaf_weight_device()
1692 	struct cfq_group *cfqg;  in __cfqg_set_weight_device()
1745 		struct cfq_group *cfqg = blkg_to_cfqg(blkg);  in __cfq_set_weight()
1825 	struct cfq_group *cfqg = pd_to_cfqg(pd);  in cfqg_prfill_avg_queue_size()
1890 		.private = offsetof(struct cfq_group, stats.time),
1895 		.private = offsetof(struct cfq_group, stats.sectors),
1900 		.private = offsetof(struct cfq_group, stats.service_bytes),
1905 		.private = offsetof(struct cfq_group, stats.serviced),
1910 		.private = offsetof(struct cfq_group, stats.service_time),
1915 		.private = offsetof(struct cfq_group, stats.wait_time),
1920 		.private = offsetof(struct cfq_group, stats.merged),
1925 		.private = offsetof(struct cfq_group, stats.queued),
1932 		.private = offsetof(struct cfq_group, stats.time),
1937 		.private = offsetof(struct cfq_group, stats.sectors),
1942 		.private = offsetof(struct cfq_group, stats.service_bytes),
1947 		.private = offsetof(struct cfq_group, stats.serviced),
1952 		.private = offsetof(struct cfq_group, stats.service_time),
1957 		.private = offsetof(struct cfq_group, stats.wait_time),
1962 		.private = offsetof(struct cfq_group, stats.merged),
1967 		.private = offsetof(struct cfq_group, stats.queued),
1977 		.private = offsetof(struct cfq_group, stats.group_wait_time),
1982 		.private = offsetof(struct cfq_group, stats.idle_time),
1987 		.private = offsetof(struct cfq_group, stats.empty_time),
1992 		.private = offsetof(struct cfq_group, stats.dequeue),
1997 		.private = offsetof(struct cfq_group, stats.unaccounted_time),
2004 static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd,  in cfq_lookup_create_cfqg()
2011 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {  in cfq_link_cfqq_cfqg()
2540 	struct cfq_group *cfqg;  in cfq_get_next_queue_forced()
2900 			struct cfq_group *cfqg, enum wl_class_t wl_class)  in cfq_choose_wl_type()
2923 choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)  in choose_wl_class_and_type()
3005 static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd)  in cfq_get_next_cfqg()
3008 	struct cfq_group *cfqg;  in cfq_get_next_cfqg()
3019 	struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);  in cfq_choose_cfqg()
3373 	struct cfq_group *cfqg;  in cfq_put_queue()
3586 	struct cfq_group *cfqg;  in cfq_find_alloc_queue()
4605 	.pd_size		= sizeof(struct cfq_group),