1#include <linux/swap_cgroup.h>
2#include <linux/vmalloc.h>
3#include <linux/mm.h>
4
5#include <linux/swapops.h> /* depends on mm.h include */
6
7static DEFINE_MUTEX(swap_cgroup_mutex);
8struct swap_cgroup_ctrl {
9	struct page **map;
10	unsigned long length;
11	spinlock_t	lock;
12};
13
14static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
15
16struct swap_cgroup {
17	unsigned short		id;
18};
19#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
20
21/*
22 * SwapCgroup implements "lookup" and "exchange" operations.
23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
24 * against SwapCache. At swap_free(), this is accessed directly from swap.
25 *
26 * This means,
27 *  - we have no race in "exchange" when we're accessed via SwapCache because
28 *    SwapCache(and its swp_entry) is under lock.
29 *  - When called via swap_free(), there is no user of this entry and no race.
30 * Then, we don't need lock around "exchange".
31 *
32 * TODO: we can push these buffers out to HIGHMEM.
33 */
34
35/*
36 * allocate buffer for swap_cgroup.
37 */
38static int swap_cgroup_prepare(int type)
39{
40	struct page *page;
41	struct swap_cgroup_ctrl *ctrl;
42	unsigned long idx, max;
43
44	ctrl = &swap_cgroup_ctrl[type];
45
46	for (idx = 0; idx < ctrl->length; idx++) {
47		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
48		if (!page)
49			goto not_enough_page;
50		ctrl->map[idx] = page;
51	}
52	return 0;
53not_enough_page:
54	max = idx;
55	for (idx = 0; idx < max; idx++)
56		__free_page(ctrl->map[idx]);
57
58	return -ENOMEM;
59}
60
61static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
62					struct swap_cgroup_ctrl **ctrlp)
63{
64	pgoff_t offset = swp_offset(ent);
65	struct swap_cgroup_ctrl *ctrl;
66	struct page *mappage;
67	struct swap_cgroup *sc;
68
69	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
70	if (ctrlp)
71		*ctrlp = ctrl;
72
73	mappage = ctrl->map[offset / SC_PER_PAGE];
74	sc = page_address(mappage);
75	return sc + offset % SC_PER_PAGE;
76}
77
78/**
79 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
80 * @ent: swap entry to be cmpxchged
81 * @old: old id
82 * @new: new id
83 *
84 * Returns old id at success, 0 at failure.
85 * (There is no mem_cgroup using 0 as its id)
86 */
87unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
88					unsigned short old, unsigned short new)
89{
90	struct swap_cgroup_ctrl *ctrl;
91	struct swap_cgroup *sc;
92	unsigned long flags;
93	unsigned short retval;
94
95	sc = lookup_swap_cgroup(ent, &ctrl);
96
97	spin_lock_irqsave(&ctrl->lock, flags);
98	retval = sc->id;
99	if (retval == old)
100		sc->id = new;
101	else
102		retval = 0;
103	spin_unlock_irqrestore(&ctrl->lock, flags);
104	return retval;
105}
106
107/**
108 * swap_cgroup_record - record mem_cgroup for this swp_entry.
109 * @ent: swap entry to be recorded into
110 * @id: mem_cgroup to be recorded
111 *
112 * Returns old value at success, 0 at failure.
113 * (Of course, old value can be 0.)
114 */
115unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
116{
117	struct swap_cgroup_ctrl *ctrl;
118	struct swap_cgroup *sc;
119	unsigned short old;
120	unsigned long flags;
121
122	sc = lookup_swap_cgroup(ent, &ctrl);
123
124	spin_lock_irqsave(&ctrl->lock, flags);
125	old = sc->id;
126	sc->id = id;
127	spin_unlock_irqrestore(&ctrl->lock, flags);
128
129	return old;
130}
131
132/**
133 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
134 * @ent: swap entry to be looked up.
135 *
136 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
137 */
138unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
139{
140	return lookup_swap_cgroup(ent, NULL)->id;
141}
142
143int swap_cgroup_swapon(int type, unsigned long max_pages)
144{
145	void *array;
146	unsigned long array_size;
147	unsigned long length;
148	struct swap_cgroup_ctrl *ctrl;
149
150	if (!do_swap_account)
151		return 0;
152
153	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
154	array_size = length * sizeof(void *);
155
156	array = vzalloc(array_size);
157	if (!array)
158		goto nomem;
159
160	ctrl = &swap_cgroup_ctrl[type];
161	mutex_lock(&swap_cgroup_mutex);
162	ctrl->length = length;
163	ctrl->map = array;
164	spin_lock_init(&ctrl->lock);
165	if (swap_cgroup_prepare(type)) {
166		/* memory shortage */
167		ctrl->map = NULL;
168		ctrl->length = 0;
169		mutex_unlock(&swap_cgroup_mutex);
170		vfree(array);
171		goto nomem;
172	}
173	mutex_unlock(&swap_cgroup_mutex);
174
175	return 0;
176nomem:
177	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
178	printk(KERN_INFO
179		"swap_cgroup can be disabled by swapaccount=0 boot option\n");
180	return -ENOMEM;
181}
182
183void swap_cgroup_swapoff(int type)
184{
185	struct page **map;
186	unsigned long i, length;
187	struct swap_cgroup_ctrl *ctrl;
188
189	if (!do_swap_account)
190		return;
191
192	mutex_lock(&swap_cgroup_mutex);
193	ctrl = &swap_cgroup_ctrl[type];
194	map = ctrl->map;
195	length = ctrl->length;
196	ctrl->map = NULL;
197	ctrl->length = 0;
198	mutex_unlock(&swap_cgroup_mutex);
199
200	if (map) {
201		for (i = 0; i < length; i++) {
202			struct page *page = map[i];
203			if (page)
204				__free_page(page);
205		}
206		vfree(map);
207	}
208}
209