861 				       struct cfq_io_cq *cic, struct bio *bio,
878 static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)  in cic_to_cfqq()  argument
880 	return cic->cfqq[is_sync];  in cic_to_cfqq()
883 static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,  in cic_set_cfqq()  argument
886 	cic->cfqq[is_sync] = cfqq;  in cic_set_cfqq()
889 static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)  in cic_to_cfqd()  argument
891 	return cic->icq.q->elevator->elevator_data;  in cic_to_cfqd()
2285 	struct cfq_io_cq *cic;  in cfq_find_rq_fmerge()  local
2288 	cic = cfq_cic_lookup(cfqd, tsk->io_context);  in cfq_find_rq_fmerge()
2289 	if (!cic)  in cfq_find_rq_fmerge()
2292 	cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));  in cfq_find_rq_fmerge()
2406 	struct cfq_io_cq *cic;  in cfq_allow_merge()  local
2419 	cic = cfq_cic_lookup(cfqd, current->io_context);  in cfq_allow_merge()
2420 	if (!cic)  in cfq_allow_merge()
2423 	cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));  in cfq_allow_merge()
2723 	struct cfq_io_cq *cic;  in cfq_arm_slice_timer()  local
2757 	cic = cfqd->active_cic;  in cfq_arm_slice_timer()
2758 	if (!cic || !atomic_read(&cic->icq.ioc->active_ref))  in cfq_arm_slice_timer()
2766 	if (sample_valid(cic->ttime.ttime_samples) &&  in cfq_arm_slice_timer()
2767 	    (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) {  in cfq_arm_slice_timer()
2769 			     cic->ttime.ttime_mean);  in cfq_arm_slice_timer()
3311 		struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_dispatch_request()  local
3313 		atomic_long_inc(&cic->icq.ioc->refcount);  in cfq_dispatch_request()
3314 		cfqd->active_cic = cic;  in cfq_dispatch_request()
3431 	struct cfq_io_cq *cic = icq_to_cic(icq);  in cfq_init_icq()  local
3433 	cic->ttime.last_end_request = jiffies;  in cfq_init_icq()
3438 	struct cfq_io_cq *cic = icq_to_cic(icq);  in cfq_exit_icq()  local
3439 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in cfq_exit_icq()
3441 	if (cic->cfqq[BLK_RW_ASYNC]) {  in cfq_exit_icq()
3442 		cfq_exit_cfqq(cfqd, cic->cfqq[BLK_RW_ASYNC]);  in cfq_exit_icq()
3443 		cic->cfqq[BLK_RW_ASYNC] = NULL;  in cfq_exit_icq()
3446 	if (cic->cfqq[BLK_RW_SYNC]) {  in cfq_exit_icq()
3447 		cfq_exit_cfqq(cfqd, cic->cfqq[BLK_RW_SYNC]);  in cfq_exit_icq()
3448 		cic->cfqq[BLK_RW_SYNC] = NULL;  in cfq_exit_icq()
3452 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)  in cfq_init_prio_data()  argument
3460 	ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);  in cfq_init_prio_data()
3472 		cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_init_prio_data()
3476 		cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_init_prio_data()
3494 static void check_ioprio_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_ioprio_changed()  argument
3496 	int ioprio = cic->icq.ioc->ioprio;  in check_ioprio_changed()
3497 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in check_ioprio_changed()
3504 	if (unlikely(!cfqd) || likely(cic->ioprio == ioprio))  in check_ioprio_changed()
3507 	cfqq = cic->cfqq[BLK_RW_ASYNC];  in check_ioprio_changed()
3510 		new_cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio,  in check_ioprio_changed()
3513 			cic->cfqq[BLK_RW_ASYNC] = new_cfqq;  in check_ioprio_changed()
3518 	cfqq = cic->cfqq[BLK_RW_SYNC];  in check_ioprio_changed()
3522 	cic->ioprio = ioprio;  in check_ioprio_changed()
3546 static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)  in check_blkcg_changed()  argument
3548 	struct cfq_data *cfqd = cic_to_cfqd(cic);  in check_blkcg_changed()
3560 	if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr))  in check_blkcg_changed()
3563 	sync_cfqq = cic_to_cfqq(cic, 1);  in check_blkcg_changed()
3570 		cic_set_cfqq(cic, NULL, 1);  in check_blkcg_changed()
3574 	cic->blkcg_serial_nr = serial_nr;  in check_blkcg_changed()
3577 static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { }  in check_blkcg_changed()  argument
3581 cfq_find_alloc_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,  in cfq_find_alloc_queue()  argument
3598 	cfqq = cic_to_cfqq(cic, is_sync);  in cfq_find_alloc_queue()
3628 			cfq_init_prio_data(cfqq, cic);  in cfq_find_alloc_queue()
3661 cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,  in cfq_get_queue()  argument
3664 	int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);  in cfq_get_queue()
3665 	int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);  in cfq_get_queue()
3670 		if (!ioprio_valid(cic->ioprio)) {  in cfq_get_queue()
3680 		cfqq = cfq_find_alloc_queue(cfqd, is_sync, cic, bio, gfp_mask);  in cfq_get_queue()
3707 			struct cfq_io_cq *cic)  in cfq_update_io_thinktime()  argument
3710 		__cfq_update_io_thinktime(&cic->ttime, cfqd->cfq_slice_idle);  in cfq_update_io_thinktime()
3745 		       struct cfq_io_cq *cic)  in cfq_update_idle_window()  argument
3762 	else if (!atomic_read(&cic->icq.ioc->active_ref) ||  in cfq_update_idle_window()
3766 	else if (sample_valid(cic->ttime.ttime_samples)) {  in cfq_update_idle_window()
3767 		if (cic->ttime.ttime_mean > cfqd->cfq_slice_idle)  in cfq_update_idle_window()
3896 	struct cfq_io_cq *cic = RQ_CIC(rq);  in cfq_rq_enqueued()  local
3902 	cfq_update_io_thinktime(cfqd, cfqq, cic);  in cfq_rq_enqueued()
3904 	cfq_update_idle_window(cfqd, cfqq, cic);  in cfq_rq_enqueued()
3997 	struct cfq_io_cq *cic = cfqd->active_cic;  in cfq_should_wait_busy()  local
4015 	if (cic && sample_valid(cic->ttime.ttime_samples)  in cfq_should_wait_busy()
4016 	    && (cfqq->slice_end - jiffies < cic->ttime.ttime_mean))  in cfq_should_wait_busy()
4134 	struct cfq_io_cq *cic;  in cfq_may_queue()  local
4143 	cic = cfq_cic_lookup(cfqd, tsk->io_context);  in cfq_may_queue()
4144 	if (!cic)  in cfq_may_queue()
4147 	cfqq = cic_to_cfqq(cic, rw_is_sync(rw));  in cfq_may_queue()
4149 		cfq_init_prio_data(cfqq, cic);  in cfq_may_queue()
4180 cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_cq *cic,  in cfq_merge_cfqqs()  argument
4184 	cic_set_cfqq(cic, cfqq->new_cfqq, 1);  in cfq_merge_cfqqs()
4187 	return cic_to_cfqq(cic, 1);  in cfq_merge_cfqqs()
4195 split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)  in split_cfqq()  argument
4204 	cic_set_cfqq(cic, NULL, 1);  in split_cfqq()
4219 	struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);  in cfq_set_request()  local
4228 	check_ioprio_changed(cic, bio);  in cfq_set_request()
4229 	check_blkcg_changed(cic, bio);  in cfq_set_request()
4231 	cfqq = cic_to_cfqq(cic, is_sync);  in cfq_set_request()
4233 		cfqq = cfq_get_queue(cfqd, is_sync, cic, bio, gfp_mask);  in cfq_set_request()
4234 		cic_set_cfqq(cic, cfqq, is_sync);  in cfq_set_request()
4241 			cfqq = split_cfqq(cic, cfqq);  in cfq_set_request()
4253 			cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq);  in cfq_set_request()