1 /*
2 * perf_event_intel_rapl.c: support Intel RAPL energy consumption counters
3 * Copyright (C) 2013 Google, Inc., Stephane Eranian
4 *
5 * Intel RAPL interface is specified in the IA-32 Manual Vol3b
6 * section 14.7.1 (September 2013)
7 *
8 * RAPL provides more controls than just reporting energy consumption
9 * however here we only expose the 3 energy consumption free running
10 * counters (pp0, pkg, dram).
11 *
12 * Each of those counters increments in a power unit defined by the
13 * RAPL_POWER_UNIT MSR. On SandyBridge, this unit is 1/(2^16) Joules
14 * but it can vary.
15 *
16 * Counter to rapl events mappings:
17 *
18 * pp0 counter: consumption of all physical cores (power plane 0)
19 * event: rapl_energy_cores
20 * perf code: 0x1
21 *
22 * pkg counter: consumption of the whole processor package
23 * event: rapl_energy_pkg
24 * perf code: 0x2
25 *
26 * dram counter: consumption of the dram domain (servers only)
27 * event: rapl_energy_dram
28 * perf code: 0x3
29 *
30 * dram counter: consumption of the builtin-gpu domain (client only)
31 * event: rapl_energy_gpu
32 * perf code: 0x4
33 *
34 * We manage those counters as free running (read-only). They may be
35 * use simultaneously by other tools, such as turbostat.
36 *
37 * The events only support system-wide mode counting. There is no
38 * sampling support because it does not make sense and is not
39 * supported by the RAPL hardware.
40 *
41 * Because we want to avoid floating-point operations in the kernel,
42 * the events are all reported in fixed point arithmetic (32.32).
43 * Tools must adjust the counts to convert them to Watts using
44 * the duration of the measurement. Tools may use a function such as
45 * ldexp(raw_count, -32);
46 */
47 #include <linux/module.h>
48 #include <linux/slab.h>
49 #include <linux/perf_event.h>
50 #include <asm/cpu_device_id.h>
51 #include "perf_event.h"
52
53 /*
54 * RAPL energy status counters
55 */
56 #define RAPL_IDX_PP0_NRG_STAT 0 /* all cores */
57 #define INTEL_RAPL_PP0 0x1 /* pseudo-encoding */
58 #define RAPL_IDX_PKG_NRG_STAT 1 /* entire package */
59 #define INTEL_RAPL_PKG 0x2 /* pseudo-encoding */
60 #define RAPL_IDX_RAM_NRG_STAT 2 /* DRAM */
61 #define INTEL_RAPL_RAM 0x3 /* pseudo-encoding */
62 #define RAPL_IDX_PP1_NRG_STAT 3 /* gpu */
63 #define INTEL_RAPL_PP1 0x4 /* pseudo-encoding */
64
65 #define NR_RAPL_DOMAINS 0x4
66 static const char *rapl_domain_names[NR_RAPL_DOMAINS] __initconst = {
67 "pp0-core",
68 "package",
69 "dram",
70 "pp1-gpu",
71 };
72
73 /* Clients have PP0, PKG */
74 #define RAPL_IDX_CLN (1<<RAPL_IDX_PP0_NRG_STAT|\
75 1<<RAPL_IDX_PKG_NRG_STAT|\
76 1<<RAPL_IDX_PP1_NRG_STAT)
77
78 /* Servers have PP0, PKG, RAM */
79 #define RAPL_IDX_SRV (1<<RAPL_IDX_PP0_NRG_STAT|\
80 1<<RAPL_IDX_PKG_NRG_STAT|\
81 1<<RAPL_IDX_RAM_NRG_STAT)
82
83 /* Servers have PP0, PKG, RAM, PP1 */
84 #define RAPL_IDX_HSW (1<<RAPL_IDX_PP0_NRG_STAT|\
85 1<<RAPL_IDX_PKG_NRG_STAT|\
86 1<<RAPL_IDX_RAM_NRG_STAT|\
87 1<<RAPL_IDX_PP1_NRG_STAT)
88
89 /*
90 * event code: LSB 8 bits, passed in attr->config
91 * any other bit is reserved
92 */
93 #define RAPL_EVENT_MASK 0xFFULL
94
95 #define DEFINE_RAPL_FORMAT_ATTR(_var, _name, _format) \
96 static ssize_t __rapl_##_var##_show(struct kobject *kobj, \
97 struct kobj_attribute *attr, \
98 char *page) \
99 { \
100 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
101 return sprintf(page, _format "\n"); \
102 } \
103 static struct kobj_attribute format_attr_##_var = \
104 __ATTR(_name, 0444, __rapl_##_var##_show, NULL)
105
106 #define RAPL_EVENT_DESC(_name, _config) \
107 { \
108 .attr = __ATTR(_name, 0444, rapl_event_show, NULL), \
109 .config = _config, \
110 }
111
112 #define RAPL_CNTR_WIDTH 32 /* 32-bit rapl counters */
113
114 #define RAPL_EVENT_ATTR_STR(_name, v, str) \
115 static struct perf_pmu_events_attr event_attr_##v = { \
116 .attr = __ATTR(_name, 0444, rapl_sysfs_show, NULL), \
117 .id = 0, \
118 .event_str = str, \
119 };
120
121 struct rapl_pmu {
122 spinlock_t lock;
123 int n_active; /* number of active events */
124 struct list_head active_list;
125 struct pmu *pmu; /* pointer to rapl_pmu_class */
126 ktime_t timer_interval; /* in ktime_t unit */
127 struct hrtimer hrtimer;
128 };
129
130 static int rapl_hw_unit[NR_RAPL_DOMAINS] __read_mostly; /* 1/2^hw_unit Joule */
131 static struct pmu rapl_pmu_class;
132 static cpumask_t rapl_cpu_mask;
133 static int rapl_cntr_mask;
134
135 static DEFINE_PER_CPU(struct rapl_pmu *, rapl_pmu);
136 static DEFINE_PER_CPU(struct rapl_pmu *, rapl_pmu_to_free);
137
138 static struct x86_pmu_quirk *rapl_quirks;
rapl_read_counter(struct perf_event * event)139 static inline u64 rapl_read_counter(struct perf_event *event)
140 {
141 u64 raw;
142 rdmsrl(event->hw.event_base, raw);
143 return raw;
144 }
145
146 #define rapl_add_quirk(func_) \
147 do { \
148 static struct x86_pmu_quirk __quirk __initdata = { \
149 .func = func_, \
150 }; \
151 __quirk.next = rapl_quirks; \
152 rapl_quirks = &__quirk; \
153 } while (0)
154
rapl_scale(u64 v,int cfg)155 static inline u64 rapl_scale(u64 v, int cfg)
156 {
157 if (cfg > NR_RAPL_DOMAINS) {
158 pr_warn("invalid domain %d, failed to scale data\n", cfg);
159 return v;
160 }
161 /*
162 * scale delta to smallest unit (1/2^32)
163 * users must then scale back: count * 1/(1e9*2^32) to get Joules
164 * or use ldexp(count, -32).
165 * Watts = Joules/Time delta
166 */
167 return v << (32 - rapl_hw_unit[cfg - 1]);
168 }
169
rapl_event_update(struct perf_event * event)170 static u64 rapl_event_update(struct perf_event *event)
171 {
172 struct hw_perf_event *hwc = &event->hw;
173 u64 prev_raw_count, new_raw_count;
174 s64 delta, sdelta;
175 int shift = RAPL_CNTR_WIDTH;
176
177 again:
178 prev_raw_count = local64_read(&hwc->prev_count);
179 rdmsrl(event->hw.event_base, new_raw_count);
180
181 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
182 new_raw_count) != prev_raw_count) {
183 cpu_relax();
184 goto again;
185 }
186
187 /*
188 * Now we have the new raw value and have updated the prev
189 * timestamp already. We can now calculate the elapsed delta
190 * (event-)time and add that to the generic event.
191 *
192 * Careful, not all hw sign-extends above the physical width
193 * of the count.
194 */
195 delta = (new_raw_count << shift) - (prev_raw_count << shift);
196 delta >>= shift;
197
198 sdelta = rapl_scale(delta, event->hw.config);
199
200 local64_add(sdelta, &event->count);
201
202 return new_raw_count;
203 }
204
rapl_start_hrtimer(struct rapl_pmu * pmu)205 static void rapl_start_hrtimer(struct rapl_pmu *pmu)
206 {
207 __hrtimer_start_range_ns(&pmu->hrtimer,
208 pmu->timer_interval, 0,
209 HRTIMER_MODE_REL_PINNED, 0);
210 }
211
rapl_stop_hrtimer(struct rapl_pmu * pmu)212 static void rapl_stop_hrtimer(struct rapl_pmu *pmu)
213 {
214 hrtimer_cancel(&pmu->hrtimer);
215 }
216
rapl_hrtimer_handle(struct hrtimer * hrtimer)217 static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
218 {
219 struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
220 struct perf_event *event;
221 unsigned long flags;
222
223 if (!pmu->n_active)
224 return HRTIMER_NORESTART;
225
226 spin_lock_irqsave(&pmu->lock, flags);
227
228 list_for_each_entry(event, &pmu->active_list, active_entry) {
229 rapl_event_update(event);
230 }
231
232 spin_unlock_irqrestore(&pmu->lock, flags);
233
234 hrtimer_forward_now(hrtimer, pmu->timer_interval);
235
236 return HRTIMER_RESTART;
237 }
238
rapl_hrtimer_init(struct rapl_pmu * pmu)239 static void rapl_hrtimer_init(struct rapl_pmu *pmu)
240 {
241 struct hrtimer *hr = &pmu->hrtimer;
242
243 hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
244 hr->function = rapl_hrtimer_handle;
245 }
246
__rapl_pmu_event_start(struct rapl_pmu * pmu,struct perf_event * event)247 static void __rapl_pmu_event_start(struct rapl_pmu *pmu,
248 struct perf_event *event)
249 {
250 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
251 return;
252
253 event->hw.state = 0;
254
255 list_add_tail(&event->active_entry, &pmu->active_list);
256
257 local64_set(&event->hw.prev_count, rapl_read_counter(event));
258
259 pmu->n_active++;
260 if (pmu->n_active == 1)
261 rapl_start_hrtimer(pmu);
262 }
263
rapl_pmu_event_start(struct perf_event * event,int mode)264 static void rapl_pmu_event_start(struct perf_event *event, int mode)
265 {
266 struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
267 unsigned long flags;
268
269 spin_lock_irqsave(&pmu->lock, flags);
270 __rapl_pmu_event_start(pmu, event);
271 spin_unlock_irqrestore(&pmu->lock, flags);
272 }
273
rapl_pmu_event_stop(struct perf_event * event,int mode)274 static void rapl_pmu_event_stop(struct perf_event *event, int mode)
275 {
276 struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
277 struct hw_perf_event *hwc = &event->hw;
278 unsigned long flags;
279
280 spin_lock_irqsave(&pmu->lock, flags);
281
282 /* mark event as deactivated and stopped */
283 if (!(hwc->state & PERF_HES_STOPPED)) {
284 WARN_ON_ONCE(pmu->n_active <= 0);
285 pmu->n_active--;
286 if (pmu->n_active == 0)
287 rapl_stop_hrtimer(pmu);
288
289 list_del(&event->active_entry);
290
291 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
292 hwc->state |= PERF_HES_STOPPED;
293 }
294
295 /* check if update of sw counter is necessary */
296 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
297 /*
298 * Drain the remaining delta count out of a event
299 * that we are disabling:
300 */
301 rapl_event_update(event);
302 hwc->state |= PERF_HES_UPTODATE;
303 }
304
305 spin_unlock_irqrestore(&pmu->lock, flags);
306 }
307
rapl_pmu_event_add(struct perf_event * event,int mode)308 static int rapl_pmu_event_add(struct perf_event *event, int mode)
309 {
310 struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
311 struct hw_perf_event *hwc = &event->hw;
312 unsigned long flags;
313
314 spin_lock_irqsave(&pmu->lock, flags);
315
316 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
317
318 if (mode & PERF_EF_START)
319 __rapl_pmu_event_start(pmu, event);
320
321 spin_unlock_irqrestore(&pmu->lock, flags);
322
323 return 0;
324 }
325
rapl_pmu_event_del(struct perf_event * event,int flags)326 static void rapl_pmu_event_del(struct perf_event *event, int flags)
327 {
328 rapl_pmu_event_stop(event, PERF_EF_UPDATE);
329 }
330
rapl_pmu_event_init(struct perf_event * event)331 static int rapl_pmu_event_init(struct perf_event *event)
332 {
333 u64 cfg = event->attr.config & RAPL_EVENT_MASK;
334 int bit, msr, ret = 0;
335
336 /* only look at RAPL events */
337 if (event->attr.type != rapl_pmu_class.type)
338 return -ENOENT;
339
340 /* check only supported bits are set */
341 if (event->attr.config & ~RAPL_EVENT_MASK)
342 return -EINVAL;
343
344 /*
345 * check event is known (determines counter)
346 */
347 switch (cfg) {
348 case INTEL_RAPL_PP0:
349 bit = RAPL_IDX_PP0_NRG_STAT;
350 msr = MSR_PP0_ENERGY_STATUS;
351 break;
352 case INTEL_RAPL_PKG:
353 bit = RAPL_IDX_PKG_NRG_STAT;
354 msr = MSR_PKG_ENERGY_STATUS;
355 break;
356 case INTEL_RAPL_RAM:
357 bit = RAPL_IDX_RAM_NRG_STAT;
358 msr = MSR_DRAM_ENERGY_STATUS;
359 break;
360 case INTEL_RAPL_PP1:
361 bit = RAPL_IDX_PP1_NRG_STAT;
362 msr = MSR_PP1_ENERGY_STATUS;
363 break;
364 default:
365 return -EINVAL;
366 }
367 /* check event supported */
368 if (!(rapl_cntr_mask & (1 << bit)))
369 return -EINVAL;
370
371 /* unsupported modes and filters */
372 if (event->attr.exclude_user ||
373 event->attr.exclude_kernel ||
374 event->attr.exclude_hv ||
375 event->attr.exclude_idle ||
376 event->attr.exclude_host ||
377 event->attr.exclude_guest ||
378 event->attr.sample_period) /* no sampling */
379 return -EINVAL;
380
381 /* must be done before validate_group */
382 event->hw.event_base = msr;
383 event->hw.config = cfg;
384 event->hw.idx = bit;
385
386 return ret;
387 }
388
rapl_pmu_event_read(struct perf_event * event)389 static void rapl_pmu_event_read(struct perf_event *event)
390 {
391 rapl_event_update(event);
392 }
393
rapl_get_attr_cpumask(struct device * dev,struct device_attribute * attr,char * buf)394 static ssize_t rapl_get_attr_cpumask(struct device *dev,
395 struct device_attribute *attr, char *buf)
396 {
397 return cpumap_print_to_pagebuf(true, buf, &rapl_cpu_mask);
398 }
399
400 static DEVICE_ATTR(cpumask, S_IRUGO, rapl_get_attr_cpumask, NULL);
401
402 static struct attribute *rapl_pmu_attrs[] = {
403 &dev_attr_cpumask.attr,
404 NULL,
405 };
406
407 static struct attribute_group rapl_pmu_attr_group = {
408 .attrs = rapl_pmu_attrs,
409 };
410
rapl_sysfs_show(struct device * dev,struct device_attribute * attr,char * page)411 static ssize_t rapl_sysfs_show(struct device *dev,
412 struct device_attribute *attr,
413 char *page)
414 {
415 struct perf_pmu_events_attr *pmu_attr = \
416 container_of(attr, struct perf_pmu_events_attr, attr);
417
418 if (pmu_attr->event_str)
419 return sprintf(page, "%s", pmu_attr->event_str);
420
421 return 0;
422 }
423
424 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
425 RAPL_EVENT_ATTR_STR(energy-pkg , rapl_pkg, "event=0x02");
426 RAPL_EVENT_ATTR_STR(energy-ram , rapl_ram, "event=0x03");
427 RAPL_EVENT_ATTR_STR(energy-gpu , rapl_gpu, "event=0x04");
428
429 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules");
430 RAPL_EVENT_ATTR_STR(energy-pkg.unit , rapl_pkg_unit, "Joules");
431 RAPL_EVENT_ATTR_STR(energy-ram.unit , rapl_ram_unit, "Joules");
432 RAPL_EVENT_ATTR_STR(energy-gpu.unit , rapl_gpu_unit, "Joules");
433
434 /*
435 * we compute in 0.23 nJ increments regardless of MSR
436 */
437 RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_cores_scale, "2.3283064365386962890625e-10");
438 RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_pkg_scale, "2.3283064365386962890625e-10");
439 RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_ram_scale, "2.3283064365386962890625e-10");
440 RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_gpu_scale, "2.3283064365386962890625e-10");
441
442 static struct attribute *rapl_events_srv_attr[] = {
443 EVENT_PTR(rapl_cores),
444 EVENT_PTR(rapl_pkg),
445 EVENT_PTR(rapl_ram),
446
447 EVENT_PTR(rapl_cores_unit),
448 EVENT_PTR(rapl_pkg_unit),
449 EVENT_PTR(rapl_ram_unit),
450
451 EVENT_PTR(rapl_cores_scale),
452 EVENT_PTR(rapl_pkg_scale),
453 EVENT_PTR(rapl_ram_scale),
454 NULL,
455 };
456
457 static struct attribute *rapl_events_cln_attr[] = {
458 EVENT_PTR(rapl_cores),
459 EVENT_PTR(rapl_pkg),
460 EVENT_PTR(rapl_gpu),
461
462 EVENT_PTR(rapl_cores_unit),
463 EVENT_PTR(rapl_pkg_unit),
464 EVENT_PTR(rapl_gpu_unit),
465
466 EVENT_PTR(rapl_cores_scale),
467 EVENT_PTR(rapl_pkg_scale),
468 EVENT_PTR(rapl_gpu_scale),
469 NULL,
470 };
471
472 static struct attribute *rapl_events_hsw_attr[] = {
473 EVENT_PTR(rapl_cores),
474 EVENT_PTR(rapl_pkg),
475 EVENT_PTR(rapl_gpu),
476 EVENT_PTR(rapl_ram),
477
478 EVENT_PTR(rapl_cores_unit),
479 EVENT_PTR(rapl_pkg_unit),
480 EVENT_PTR(rapl_gpu_unit),
481 EVENT_PTR(rapl_ram_unit),
482
483 EVENT_PTR(rapl_cores_scale),
484 EVENT_PTR(rapl_pkg_scale),
485 EVENT_PTR(rapl_gpu_scale),
486 EVENT_PTR(rapl_ram_scale),
487 NULL,
488 };
489
490 static struct attribute_group rapl_pmu_events_group = {
491 .name = "events",
492 .attrs = NULL, /* patched at runtime */
493 };
494
495 DEFINE_RAPL_FORMAT_ATTR(event, event, "config:0-7");
496 static struct attribute *rapl_formats_attr[] = {
497 &format_attr_event.attr,
498 NULL,
499 };
500
501 static struct attribute_group rapl_pmu_format_group = {
502 .name = "format",
503 .attrs = rapl_formats_attr,
504 };
505
506 const struct attribute_group *rapl_attr_groups[] = {
507 &rapl_pmu_attr_group,
508 &rapl_pmu_format_group,
509 &rapl_pmu_events_group,
510 NULL,
511 };
512
513 static struct pmu rapl_pmu_class = {
514 .attr_groups = rapl_attr_groups,
515 .task_ctx_nr = perf_invalid_context, /* system-wide only */
516 .event_init = rapl_pmu_event_init,
517 .add = rapl_pmu_event_add, /* must have */
518 .del = rapl_pmu_event_del, /* must have */
519 .start = rapl_pmu_event_start,
520 .stop = rapl_pmu_event_stop,
521 .read = rapl_pmu_event_read,
522 };
523
rapl_cpu_exit(int cpu)524 static void rapl_cpu_exit(int cpu)
525 {
526 struct rapl_pmu *pmu = per_cpu(rapl_pmu, cpu);
527 int i, phys_id = topology_physical_package_id(cpu);
528 int target = -1;
529
530 /* find a new cpu on same package */
531 for_each_online_cpu(i) {
532 if (i == cpu)
533 continue;
534 if (phys_id == topology_physical_package_id(i)) {
535 target = i;
536 break;
537 }
538 }
539 /*
540 * clear cpu from cpumask
541 * if was set in cpumask and still some cpu on package,
542 * then move to new cpu
543 */
544 if (cpumask_test_and_clear_cpu(cpu, &rapl_cpu_mask) && target >= 0)
545 cpumask_set_cpu(target, &rapl_cpu_mask);
546
547 WARN_ON(cpumask_empty(&rapl_cpu_mask));
548 /*
549 * migrate events and context to new cpu
550 */
551 if (target >= 0)
552 perf_pmu_migrate_context(pmu->pmu, cpu, target);
553
554 /* cancel overflow polling timer for CPU */
555 rapl_stop_hrtimer(pmu);
556 }
557
rapl_cpu_init(int cpu)558 static void rapl_cpu_init(int cpu)
559 {
560 int i, phys_id = topology_physical_package_id(cpu);
561
562 /* check if phys_is is already covered */
563 for_each_cpu(i, &rapl_cpu_mask) {
564 if (phys_id == topology_physical_package_id(i))
565 return;
566 }
567 /* was not found, so add it */
568 cpumask_set_cpu(cpu, &rapl_cpu_mask);
569 }
570
rapl_hsw_server_quirk(void)571 static __init void rapl_hsw_server_quirk(void)
572 {
573 /*
574 * DRAM domain on HSW server has fixed energy unit which can be
575 * different than the unit from power unit MSR.
576 * "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, V2
577 * of 2. Datasheet, September 2014, Reference Number: 330784-001 "
578 */
579 rapl_hw_unit[RAPL_IDX_RAM_NRG_STAT] = 16;
580 }
581
rapl_cpu_prepare(int cpu)582 static int rapl_cpu_prepare(int cpu)
583 {
584 struct rapl_pmu *pmu = per_cpu(rapl_pmu, cpu);
585 int phys_id = topology_physical_package_id(cpu);
586 u64 ms;
587
588 if (pmu)
589 return 0;
590
591 if (phys_id < 0)
592 return -1;
593
594 pmu = kzalloc_node(sizeof(*pmu), GFP_KERNEL, cpu_to_node(cpu));
595 if (!pmu)
596 return -1;
597 spin_lock_init(&pmu->lock);
598
599 INIT_LIST_HEAD(&pmu->active_list);
600
601 pmu->pmu = &rapl_pmu_class;
602
603 /*
604 * use reference of 200W for scaling the timeout
605 * to avoid missing counter overflows.
606 * 200W = 200 Joules/sec
607 * divide interval by 2 to avoid lockstep (2 * 100)
608 * if hw unit is 32, then we use 2 ms 1/200/2
609 */
610 if (rapl_hw_unit[0] < 32)
611 ms = (1000 / (2 * 100)) * (1ULL << (32 - rapl_hw_unit[0] - 1));
612 else
613 ms = 2;
614
615 pmu->timer_interval = ms_to_ktime(ms);
616
617 rapl_hrtimer_init(pmu);
618
619 /* set RAPL pmu for this cpu for now */
620 per_cpu(rapl_pmu, cpu) = pmu;
621 per_cpu(rapl_pmu_to_free, cpu) = NULL;
622
623 return 0;
624 }
625
rapl_cpu_kfree(int cpu)626 static void rapl_cpu_kfree(int cpu)
627 {
628 struct rapl_pmu *pmu = per_cpu(rapl_pmu_to_free, cpu);
629
630 kfree(pmu);
631
632 per_cpu(rapl_pmu_to_free, cpu) = NULL;
633 }
634
rapl_cpu_dying(int cpu)635 static int rapl_cpu_dying(int cpu)
636 {
637 struct rapl_pmu *pmu = per_cpu(rapl_pmu, cpu);
638
639 if (!pmu)
640 return 0;
641
642 per_cpu(rapl_pmu, cpu) = NULL;
643
644 per_cpu(rapl_pmu_to_free, cpu) = pmu;
645
646 return 0;
647 }
648
rapl_cpu_notifier(struct notifier_block * self,unsigned long action,void * hcpu)649 static int rapl_cpu_notifier(struct notifier_block *self,
650 unsigned long action, void *hcpu)
651 {
652 unsigned int cpu = (long)hcpu;
653
654 switch (action & ~CPU_TASKS_FROZEN) {
655 case CPU_UP_PREPARE:
656 rapl_cpu_prepare(cpu);
657 break;
658 case CPU_STARTING:
659 rapl_cpu_init(cpu);
660 break;
661 case CPU_UP_CANCELED:
662 case CPU_DYING:
663 rapl_cpu_dying(cpu);
664 break;
665 case CPU_ONLINE:
666 case CPU_DEAD:
667 rapl_cpu_kfree(cpu);
668 break;
669 case CPU_DOWN_PREPARE:
670 rapl_cpu_exit(cpu);
671 break;
672 default:
673 break;
674 }
675
676 return NOTIFY_OK;
677 }
678
rapl_check_hw_unit(void)679 static int rapl_check_hw_unit(void)
680 {
681 u64 msr_rapl_power_unit_bits;
682 int i;
683
684 /* protect rdmsrl() to handle virtualization */
685 if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &msr_rapl_power_unit_bits))
686 return -1;
687 for (i = 0; i < NR_RAPL_DOMAINS; i++)
688 rapl_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
689
690 return 0;
691 }
692
693 static const struct x86_cpu_id rapl_cpu_match[] = {
694 [0] = { .vendor = X86_VENDOR_INTEL, .family = 6 },
695 [1] = {},
696 };
697
rapl_pmu_init(void)698 static int __init rapl_pmu_init(void)
699 {
700 struct rapl_pmu *pmu;
701 int cpu, ret;
702 struct x86_pmu_quirk *quirk;
703 int i;
704
705 /*
706 * check for Intel processor family 6
707 */
708 if (!x86_match_cpu(rapl_cpu_match))
709 return 0;
710
711 /* check supported CPU */
712 switch (boot_cpu_data.x86_model) {
713 case 42: /* Sandy Bridge */
714 case 58: /* Ivy Bridge */
715 rapl_cntr_mask = RAPL_IDX_CLN;
716 rapl_pmu_events_group.attrs = rapl_events_cln_attr;
717 break;
718 case 63: /* Haswell-Server */
719 rapl_add_quirk(rapl_hsw_server_quirk);
720 rapl_cntr_mask = RAPL_IDX_SRV;
721 rapl_pmu_events_group.attrs = rapl_events_srv_attr;
722 break;
723 case 60: /* Haswell */
724 case 69: /* Haswell-Celeron */
725 case 61: /* Broadwell */
726 rapl_cntr_mask = RAPL_IDX_HSW;
727 rapl_pmu_events_group.attrs = rapl_events_hsw_attr;
728 break;
729 case 45: /* Sandy Bridge-EP */
730 case 62: /* IvyTown */
731 rapl_cntr_mask = RAPL_IDX_SRV;
732 rapl_pmu_events_group.attrs = rapl_events_srv_attr;
733 break;
734
735 default:
736 /* unsupported */
737 return 0;
738 }
739 ret = rapl_check_hw_unit();
740 if (ret)
741 return ret;
742
743 /* run cpu model quirks */
744 for (quirk = rapl_quirks; quirk; quirk = quirk->next)
745 quirk->func();
746 cpu_notifier_register_begin();
747
748 for_each_online_cpu(cpu) {
749 ret = rapl_cpu_prepare(cpu);
750 if (ret)
751 goto out;
752 rapl_cpu_init(cpu);
753 }
754
755 __perf_cpu_notifier(rapl_cpu_notifier);
756
757 ret = perf_pmu_register(&rapl_pmu_class, "power", -1);
758 if (WARN_ON(ret)) {
759 pr_info("RAPL PMU detected, registration failed (%d), RAPL PMU disabled\n", ret);
760 cpu_notifier_register_done();
761 return -1;
762 }
763
764 pmu = __this_cpu_read(rapl_pmu);
765
766 pr_info("RAPL PMU detected,"
767 " API unit is 2^-32 Joules,"
768 " %d fixed counters"
769 " %llu ms ovfl timer\n",
770 hweight32(rapl_cntr_mask),
771 ktime_to_ms(pmu->timer_interval));
772 for (i = 0; i < NR_RAPL_DOMAINS; i++) {
773 if (rapl_cntr_mask & (1 << i)) {
774 pr_info("hw unit of domain %s 2^-%d Joules\n",
775 rapl_domain_names[i], rapl_hw_unit[i]);
776 }
777 }
778 out:
779 cpu_notifier_register_done();
780
781 return 0;
782 }
783 device_initcall(rapl_pmu_init);
784