55 	struct throtl_grp	*tg;		/* tg this qnode belongs to */
84 #define rb_entry_tg(node)	rb_entry((node), struct throtl_grp, rb_node)
94 struct throtl_grp {  struct
180 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)  in pd_to_tg()
182 	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;  in pd_to_tg()
185 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)  in blkg_to_tg()
190 static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)  in tg_to_blkg()
195 static inline struct throtl_grp *td_root_tg(struct throtl_data *td)  in td_root_tg()
207 static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)  in sq_to_tg()
210 		return container_of(sq, struct throtl_grp, service_queue);  in sq_to_tg()
224 	struct throtl_grp *tg = sq_to_tg(sq);  in sq_to_td()
245 	struct throtl_grp *__tg = sq_to_tg((sq));			\
293 		struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,  in tg_stats_alloc_fn()
294 							 struct throtl_grp,  in tg_stats_alloc_fn()
306 static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg)  in throtl_qnode_init()
365 				     struct throtl_grp **tg_to_put)  in throtl_pop_queued()
408 	struct throtl_grp *tg = blkg_to_tg(blkg);  in throtl_pd_init()
463 static void tg_update_has_rules(struct throtl_grp *tg)  in tg_update_has_rules()
465 	struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq);  in tg_update_has_rules()
484 	struct throtl_grp *tg = blkg_to_tg(blkg);  in throtl_pd_exit()
498 	struct throtl_grp *tg = blkg_to_tg(blkg);  in throtl_pd_reset_stats()
512 static struct throtl_grp *throtl_lookup_tg(struct throtl_data *td,  in throtl_lookup_tg()
525 static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td,  in throtl_lookup_create_tg()
529 	struct throtl_grp *tg = NULL;  in throtl_lookup_create_tg()
552 static struct throtl_grp *
585 	struct throtl_grp *tg;  in update_min_dispatch_time()
594 static void tg_service_queue_add(struct throtl_grp *tg)  in tg_service_queue_add()
599 	struct throtl_grp *__tg;  in tg_service_queue_add()
622 static void __throtl_enqueue_tg(struct throtl_grp *tg)  in __throtl_enqueue_tg()
629 static void throtl_enqueue_tg(struct throtl_grp *tg)  in throtl_enqueue_tg()
635 static void __throtl_dequeue_tg(struct throtl_grp *tg)  in __throtl_dequeue_tg()
641 static void throtl_dequeue_tg(struct throtl_grp *tg)  in throtl_dequeue_tg()
693 static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg,  in throtl_start_new_slice_with_credit()
715 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)  in throtl_start_new_slice()
727 static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,  in throtl_set_slice_end()
733 static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,  in throtl_extend_slice()
744 static bool throtl_slice_used(struct throtl_grp *tg, bool rw)  in throtl_slice_used()
753 static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)  in throtl_trim_slice()
811 static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,  in tg_with_in_iops_limit()
861 static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,  in tg_with_in_bps_limit()
907 static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,  in tg_may_dispatch()
962 	struct throtl_grp *tg = blkg_to_tg(blkg);  in throtl_update_dispatch_stats()
985 static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)  in throtl_charge_bio()
1021 			      struct throtl_grp *tg)  in throtl_add_bio_tg()
1044 static void tg_update_disptime(struct throtl_grp *tg)  in tg_update_disptime()
1068 static void start_parent_slice_with_credit(struct throtl_grp *child_tg,  in start_parent_slice_with_credit()
1069 					struct throtl_grp *parent_tg, bool rw)  in start_parent_slice_with_credit()
1078 static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)  in tg_dispatch_one_bio()
1082 	struct throtl_grp *parent_tg = sq_to_tg(parent_sq);  in tg_dispatch_one_bio()
1083 	struct throtl_grp *tg_to_put = NULL;  in tg_dispatch_one_bio()
1120 static int throtl_dispatch_tg(struct throtl_grp *tg)  in throtl_dispatch_tg()
1158 		struct throtl_grp *tg = throtl_rb_first(parent_sq);  in throtl_select_dispatch()
1199 	struct throtl_grp *tg = sq_to_tg(sq);  in throtl_pending_timer_fn()
1291 	struct throtl_grp *tg = pd_to_tg(pd);  in tg_prfill_cpu_rwstat()
1319 	struct throtl_grp *tg = pd_to_tg(pd);  in tg_prfill_conf_u64()
1330 	struct throtl_grp *tg = pd_to_tg(pd);  in tg_prfill_conf_uint()
1357 	struct throtl_grp *tg;  in tg_set_conf()
1428 		.private = offsetof(struct throtl_grp, bps[READ]),
1434 		.private = offsetof(struct throtl_grp, bps[WRITE]),
1440 		.private = offsetof(struct throtl_grp, iops[READ]),
1446 		.private = offsetof(struct throtl_grp, iops[WRITE]),
1471 	.pd_size		= sizeof(struct throtl_grp),
1484 	struct throtl_grp *tg;  in blk_throtl_bio()
1603 	struct throtl_grp *tg;  in tg_drain_bios()