254 static void q_init(struct queue *q, struct entry_space *es, unsigned nr_levels)  in q_init()  argument
258 	q->es = es;  in q_init()
259 	q->nr_elts = 0;  in q_init()
260 	q->nr_levels = nr_levels;  in q_init()
262 	for (i = 0; i < q->nr_levels; i++) {  in q_init()
263 		l_init(q->qs + i);  in q_init()
264 		q->target_count[i] = 0u;  in q_init()
267 	q->last_target_nr_elts = 0u;  in q_init()
268 	q->nr_top_levels = 0u;  in q_init()
269 	q->nr_in_top_levels = 0u;  in q_init()
272 static unsigned q_size(struct queue *q)  in q_size()  argument
274 	return q->nr_elts;  in q_size()
280 static void q_push(struct queue *q, struct entry *e)  in q_push()  argument
283 		q->nr_elts++;  in q_push()
285 	l_add_tail(q->es, q->qs + e->level, e);  in q_push()
288 static void q_push_before(struct queue *q, struct entry *old, struct entry *e)  in q_push_before()  argument
291 		q->nr_elts++;  in q_push_before()
293 	l_add_before(q->es, q->qs + e->level, old, e);  in q_push_before()
296 static void q_del(struct queue *q, struct entry *e)  in q_del()  argument
298 	l_del(q->es, q->qs + e->level, e);  in q_del()
300 		q->nr_elts--;  in q_del()
306 static struct entry *q_peek(struct queue *q, unsigned max_level, bool can_cross_sentinel)  in q_peek()  argument
311 	max_level = min(max_level, q->nr_levels);  in q_peek()
314 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e)) {  in q_peek()
328 static struct entry *q_pop(struct queue *q)  in q_pop()  argument
330 	struct entry *e = q_peek(q, q->nr_levels, true);  in q_pop()
333 		q_del(q, e);  in q_pop()
341 static struct entry *q_pop_old(struct queue *q, unsigned max_level)  in q_pop_old()  argument
343 	struct entry *e = q_peek(q, max_level, false);  in q_pop_old()
346 		q_del(q, e);  in q_pop_old()
356 static struct entry *__redist_pop_from(struct queue *q, unsigned level)  in __redist_pop_from()  argument
360 	for (; level < q->nr_levels; level++)  in __redist_pop_from()
361 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e))  in __redist_pop_from()
363 				l_del(q->es, q->qs + e->level, e);  in __redist_pop_from()
370 static void q_set_targets_subrange_(struct queue *q, unsigned nr_elts, unsigned lbegin, unsigned le…  in q_set_targets_subrange_()  argument
375 	BUG_ON(lend > q->nr_levels);  in q_set_targets_subrange_()
381 		q->target_count[level] =  in q_set_targets_subrange_()
389 static void q_set_targets(struct queue *q)  in q_set_targets()  argument
391 	if (q->last_target_nr_elts == q->nr_elts)  in q_set_targets()
394 	q->last_target_nr_elts = q->nr_elts;  in q_set_targets()
396 	if (q->nr_top_levels > q->nr_levels)  in q_set_targets()
397 		q_set_targets_subrange_(q, q->nr_elts, 0, q->nr_levels);  in q_set_targets()
400 		q_set_targets_subrange_(q, q->nr_in_top_levels,  in q_set_targets()
401 					q->nr_levels - q->nr_top_levels, q->nr_levels);  in q_set_targets()
403 		if (q->nr_in_top_levels < q->nr_elts)  in q_set_targets()
404 			q_set_targets_subrange_(q, q->nr_elts - q->nr_in_top_levels,  in q_set_targets()
405 						0, q->nr_levels - q->nr_top_levels);  in q_set_targets()
407 			q_set_targets_subrange_(q, 0, 0, q->nr_levels - q->nr_top_levels);  in q_set_targets()
411 static void q_redistribute(struct queue *q)  in q_redistribute()  argument
417 	q_set_targets(q);  in q_redistribute()
419 	for (level = 0u; level < q->nr_levels - 1u; level++) {  in q_redistribute()
420 		l = q->qs + level;  in q_redistribute()
421 		target = q->target_count[level];  in q_redistribute()
427 			e = __redist_pop_from(q, level + 1u);  in q_redistribute()
434 			l_add_tail(q->es, l, e);  in q_redistribute()
440 		l_above = q->qs + level + 1u;  in q_redistribute()
442 			e = l_pop_tail(q->es, l);  in q_redistribute()
449 			l_add_head(q->es, l_above, e);  in q_redistribute()
454 static void q_requeue_before(struct queue *q, struct entry *dest, struct entry *e, unsigned extra_l…  in q_requeue_before()  argument
459 	q_del(q, e);  in q_requeue_before()
461 	if (extra_levels && (e->level < q->nr_levels - 1u)) {  in q_requeue_before()
462 		new_level = min(q->nr_levels - 1u, e->level + extra_levels);  in q_requeue_before()
463 		for (de = l_head(q->es, q->qs + new_level); de; de = l_next(q->es, de)) {  in q_requeue_before()
467 			q_del(q, de);  in q_requeue_before()
471 				q_push_before(q, dest, de);  in q_requeue_before()
473 				q_push(q, de);  in q_requeue_before()
480 	q_push(q, e);  in q_requeue_before()
483 static void q_requeue(struct queue *q, struct entry *e, unsigned extra_levels)  in q_requeue()  argument
485 	q_requeue_before(q, NULL, e, extra_levels);  in q_requeue()
853 	struct queue *q = &mq->dirty;  in __update_writeback_sentinels()  local
856 	for (level = 0; level < q->nr_levels; level++) {  in __update_writeback_sentinels()
858 		q_del(q, sentinel);  in __update_writeback_sentinels()
859 		q_push(q, sentinel);  in __update_writeback_sentinels()
866 	struct queue *q = &mq->clean;  in __update_demote_sentinels()  local
869 	for (level = 0; level < q->nr_levels; level++) {  in __update_demote_sentinels()
871 		q_del(q, sentinel);  in __update_demote_sentinels()
872 		q_push(q, sentinel);  in __update_demote_sentinels()
928 	struct queue *q = e->dirty ? &mq->dirty : &mq->clean;  in push_new()  local
930 	q_push(q, e);  in push_new()
955 static void __del(struct smq_policy *mq, struct queue *q, struct entry *e)  in __del()  argument
957 	q_del(q, e);  in __del()
966 static struct entry *pop_old(struct smq_policy *mq, struct queue *q, unsigned max_level)  in pop_old()  argument
968 	struct entry *e = q_pop_old(q, max_level);  in pop_old()
1378 static int smq_save_hints(struct smq_policy *mq, struct queue *q,  in smq_save_hints()  argument
1385 	for (level = 0; level < q->nr_levels; level++)  in smq_save_hints()
1386 		for (e = l_head(q->es, q->qs + level); e; e = l_next(q->es, e)) {  in smq_save_hints()