49 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
55 	lockdep_assert_held(&nlru->lock);  in list_lru_from_memcg_idx()
56 	if (nlru->memcg_lrus && idx >= 0)  in list_lru_from_memcg_idx()
57 		return nlru->memcg_lrus->lru[idx];  in list_lru_from_memcg_idx()
59 	return &nlru->lru;  in list_lru_from_memcg_idx()
63 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)  in list_lru_from_kmem()  argument
67 	if (!nlru->memcg_lrus)  in list_lru_from_kmem()
68 		return &nlru->lru;  in list_lru_from_kmem()
72 		return &nlru->lru;  in list_lru_from_kmem()
74 	return list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));  in list_lru_from_kmem()
83 list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)  in list_lru_from_memcg_idx()  argument
85 	return &nlru->lru;  in list_lru_from_memcg_idx()
89 list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)  in list_lru_from_kmem()  argument
91 	return &nlru->lru;  in list_lru_from_kmem()
98 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_add()  local
101 	spin_lock(&nlru->lock);  in list_lru_add()
102 	l = list_lru_from_kmem(nlru, item);  in list_lru_add()
106 		spin_unlock(&nlru->lock);  in list_lru_add()
109 	spin_unlock(&nlru->lock);  in list_lru_add()
117 	struct list_lru_node *nlru = &lru->node[nid];  in list_lru_del()  local
120 	spin_lock(&nlru->lock);  in list_lru_del()
121 	l = list_lru_from_kmem(nlru, item);  in list_lru_del()
125 		spin_unlock(&nlru->lock);  in list_lru_del()
128 	spin_unlock(&nlru->lock);  in list_lru_del()
151 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_count_one()  local
155 	spin_lock(&nlru->lock);  in __list_lru_count_one()
156 	l = list_lru_from_memcg_idx(nlru, memcg_idx);  in __list_lru_count_one()
158 	spin_unlock(&nlru->lock);  in __list_lru_count_one()
190 	struct list_lru_node *nlru = &lru->node[nid];  in __list_lru_walk_one()  local
195 	spin_lock(&nlru->lock);  in __list_lru_walk_one()
196 	l = list_lru_from_memcg_idx(nlru, memcg_idx);  in __list_lru_walk_one()
209 		ret = isolate(item, l, &nlru->lock, cb_arg);  in __list_lru_walk_one()
212 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
233 			assert_spin_locked(&nlru->lock);  in __list_lru_walk_one()
240 	spin_unlock(&nlru->lock);  in __list_lru_walk_one()
312 static int memcg_init_list_lru_node(struct list_lru_node *nlru)  in memcg_init_list_lru_node()  argument
316 	nlru->memcg_lrus = kmalloc(size * sizeof(void *), GFP_KERNEL);  in memcg_init_list_lru_node()
317 	if (!nlru->memcg_lrus)  in memcg_init_list_lru_node()
320 	if (__memcg_init_list_lru_node(nlru->memcg_lrus, 0, size)) {  in memcg_init_list_lru_node()
321 		kfree(nlru->memcg_lrus);  in memcg_init_list_lru_node()
328 static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)  in memcg_destroy_list_lru_node()  argument
330 	__memcg_destroy_list_lru_node(nlru->memcg_lrus, 0, memcg_nr_cache_ids);  in memcg_destroy_list_lru_node()
331 	kfree(nlru->memcg_lrus);  in memcg_destroy_list_lru_node()
334 static int memcg_update_list_lru_node(struct list_lru_node *nlru,  in memcg_update_list_lru_node()  argument
341 	old = nlru->memcg_lrus;  in memcg_update_list_lru_node()
360 	spin_lock_irq(&nlru->lock);  in memcg_update_list_lru_node()
361 	nlru->memcg_lrus = new;  in memcg_update_list_lru_node()
362 	spin_unlock_irq(&nlru->lock);  in memcg_update_list_lru_node()
368 static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,  in memcg_cancel_update_list_lru_node()  argument
373 	__memcg_destroy_list_lru_node(nlru->memcg_lrus, old_size, new_size);  in memcg_cancel_update_list_lru_node()
459 static void memcg_drain_list_lru_node(struct list_lru_node *nlru,  in memcg_drain_list_lru_node()  argument
468 	spin_lock_irq(&nlru->lock);  in memcg_drain_list_lru_node()
470 	src = list_lru_from_memcg_idx(nlru, src_idx);  in memcg_drain_list_lru_node()
471 	dst = list_lru_from_memcg_idx(nlru, dst_idx);  in memcg_drain_list_lru_node()
477 	spin_unlock_irq(&nlru->lock);  in memcg_drain_list_lru_node()