1/* 2 * Copyright 2014 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 * 14 * 15 * Perf_events support for Tile processor. 16 * 17 * This code is based upon the x86 perf event 18 * code, which is: 19 * 20 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 21 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 22 * Copyright (C) 2009 Jaswinder Singh Rajput 23 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 24 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 25 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 26 * Copyright (C) 2009 Google, Inc., Stephane Eranian 27 */ 28 29#include <linux/kprobes.h> 30#include <linux/kernel.h> 31#include <linux/kdebug.h> 32#include <linux/mutex.h> 33#include <linux/bitmap.h> 34#include <linux/irq.h> 35#include <linux/interrupt.h> 36#include <linux/perf_event.h> 37#include <linux/atomic.h> 38#include <asm/traps.h> 39#include <asm/stack.h> 40#include <asm/pmc.h> 41#include <hv/hypervisor.h> 42 43#define TILE_MAX_COUNTERS 4 44 45#define PERF_COUNT_0_IDX 0 46#define PERF_COUNT_1_IDX 1 47#define AUX_PERF_COUNT_0_IDX 2 48#define AUX_PERF_COUNT_1_IDX 3 49 50struct cpu_hw_events { 51 int n_events; 52 struct perf_event *events[TILE_MAX_COUNTERS]; /* counter order */ 53 struct perf_event *event_list[TILE_MAX_COUNTERS]; /* enabled 54 order */ 55 int assign[TILE_MAX_COUNTERS]; 56 unsigned long active_mask[BITS_TO_LONGS(TILE_MAX_COUNTERS)]; 57 unsigned long used_mask; 58}; 59 60/* TILE arch specific performance monitor unit */ 61struct tile_pmu { 62 const char *name; 63 int version; 64 const int *hw_events; /* generic hw events table */ 65 /* generic hw cache events table */ 66 const int (*cache_events)[PERF_COUNT_HW_CACHE_MAX] 67 [PERF_COUNT_HW_CACHE_OP_MAX] 68 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 69 int (*map_hw_event)(u64); /*method used to map 70 hw events */ 71 int (*map_cache_event)(u64); /*method used to map 72 cache events */ 73 74 u64 max_period; /* max sampling period */ 75 u64 cntval_mask; /* counter width mask */ 76 int cntval_bits; /* counter width */ 77 int max_events; /* max generic hw events 78 in map */ 79 int num_counters; /* number base + aux counters */ 80 int num_base_counters; /* number base counters */ 81}; 82 83DEFINE_PER_CPU(u64, perf_irqs); 84static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); 85 86#define TILE_OP_UNSUPP (-1) 87 88#ifndef __tilegx__ 89/* TILEPro hardware events map */ 90static const int tile_hw_event_map[] = { 91 [PERF_COUNT_HW_CPU_CYCLES] = 0x01, /* ONE */ 92 [PERF_COUNT_HW_INSTRUCTIONS] = 0x06, /* MP_BUNDLE_RETIRED */ 93 [PERF_COUNT_HW_CACHE_REFERENCES] = TILE_OP_UNSUPP, 94 [PERF_COUNT_HW_CACHE_MISSES] = TILE_OP_UNSUPP, 95 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x16, /* 96 MP_CONDITIONAL_BRANCH_ISSUED */ 97 [PERF_COUNT_HW_BRANCH_MISSES] = 0x14, /* 98 MP_CONDITIONAL_BRANCH_MISSPREDICT */ 99 [PERF_COUNT_HW_BUS_CYCLES] = TILE_OP_UNSUPP, 100}; 101#else 102/* TILEGx hardware events map */ 103static const int tile_hw_event_map[] = { 104 [PERF_COUNT_HW_CPU_CYCLES] = 0x181, /* ONE */ 105 [PERF_COUNT_HW_INSTRUCTIONS] = 0xdb, /* INSTRUCTION_BUNDLE */ 106 [PERF_COUNT_HW_CACHE_REFERENCES] = TILE_OP_UNSUPP, 107 [PERF_COUNT_HW_CACHE_MISSES] = TILE_OP_UNSUPP, 108 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0xd9, /* 109 COND_BRANCH_PRED_CORRECT */ 110 [PERF_COUNT_HW_BRANCH_MISSES] = 0xda, /* 111 COND_BRANCH_PRED_INCORRECT */ 112 [PERF_COUNT_HW_BUS_CYCLES] = TILE_OP_UNSUPP, 113}; 114#endif 115 116#define C(x) PERF_COUNT_HW_CACHE_##x 117 118/* 119 * Generalized hw caching related hw_event table, filled 120 * in on a per model basis. A value of -1 means 121 * 'not supported', any other value means the 122 * raw hw_event ID. 123 */ 124#ifndef __tilegx__ 125/* TILEPro hardware cache event map */ 126static const int tile_cache_event_map[PERF_COUNT_HW_CACHE_MAX] 127 [PERF_COUNT_HW_CACHE_OP_MAX] 128 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 129[C(L1D)] = { 130 [C(OP_READ)] = { 131 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 132 [C(RESULT_MISS)] = 0x21, /* RD_MISS */ 133 }, 134 [C(OP_WRITE)] = { 135 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 136 [C(RESULT_MISS)] = 0x22, /* WR_MISS */ 137 }, 138 [C(OP_PREFETCH)] = { 139 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 140 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 141 }, 142}, 143[C(L1I)] = { 144 [C(OP_READ)] = { 145 [C(RESULT_ACCESS)] = 0x12, /* MP_ICACHE_HIT_ISSUED */ 146 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 147 }, 148 [C(OP_WRITE)] = { 149 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 150 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 151 }, 152 [C(OP_PREFETCH)] = { 153 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 154 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 155 }, 156}, 157[C(LL)] = { 158 [C(OP_READ)] = { 159 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 160 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 161 }, 162 [C(OP_WRITE)] = { 163 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 164 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 165 }, 166 [C(OP_PREFETCH)] = { 167 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 168 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 169 }, 170}, 171[C(DTLB)] = { 172 [C(OP_READ)] = { 173 [C(RESULT_ACCESS)] = 0x1d, /* TLB_CNT */ 174 [C(RESULT_MISS)] = 0x20, /* TLB_EXCEPTION */ 175 }, 176 [C(OP_WRITE)] = { 177 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 178 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 179 }, 180 [C(OP_PREFETCH)] = { 181 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 182 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 183 }, 184}, 185[C(ITLB)] = { 186 [C(OP_READ)] = { 187 [C(RESULT_ACCESS)] = 0x13, /* MP_ITLB_HIT_ISSUED */ 188 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 189 }, 190 [C(OP_WRITE)] = { 191 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 192 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 193 }, 194 [C(OP_PREFETCH)] = { 195 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 196 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 197 }, 198}, 199[C(BPU)] = { 200 [C(OP_READ)] = { 201 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 202 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 203 }, 204 [C(OP_WRITE)] = { 205 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 206 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 207 }, 208 [C(OP_PREFETCH)] = { 209 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 210 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 211 }, 212}, 213}; 214#else 215/* TILEGx hardware events map */ 216static const int tile_cache_event_map[PERF_COUNT_HW_CACHE_MAX] 217 [PERF_COUNT_HW_CACHE_OP_MAX] 218 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 219[C(L1D)] = { 220 /* 221 * Like some other architectures (e.g. ARM), the performance 222 * counters don't differentiate between read and write 223 * accesses/misses, so this isn't strictly correct, but it's the 224 * best we can do. Writes and reads get combined. 225 */ 226 [C(OP_READ)] = { 227 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 228 [C(RESULT_MISS)] = 0x44, /* RD_MISS */ 229 }, 230 [C(OP_WRITE)] = { 231 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 232 [C(RESULT_MISS)] = 0x45, /* WR_MISS */ 233 }, 234 [C(OP_PREFETCH)] = { 235 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 236 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 237 }, 238}, 239[C(L1I)] = { 240 [C(OP_READ)] = { 241 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 242 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 243 }, 244 [C(OP_WRITE)] = { 245 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 246 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 247 }, 248 [C(OP_PREFETCH)] = { 249 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 250 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 251 }, 252}, 253[C(LL)] = { 254 [C(OP_READ)] = { 255 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 256 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 257 }, 258 [C(OP_WRITE)] = { 259 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 260 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 261 }, 262 [C(OP_PREFETCH)] = { 263 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 264 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 265 }, 266}, 267[C(DTLB)] = { 268 [C(OP_READ)] = { 269 [C(RESULT_ACCESS)] = 0x40, /* TLB_CNT */ 270 [C(RESULT_MISS)] = 0x43, /* TLB_EXCEPTION */ 271 }, 272 [C(OP_WRITE)] = { 273 [C(RESULT_ACCESS)] = 0x40, /* TLB_CNT */ 274 [C(RESULT_MISS)] = 0x43, /* TLB_EXCEPTION */ 275 }, 276 [C(OP_PREFETCH)] = { 277 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 278 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 279 }, 280}, 281[C(ITLB)] = { 282 [C(OP_READ)] = { 283 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 284 [C(RESULT_MISS)] = 0xd4, /* ITLB_MISS_INT */ 285 }, 286 [C(OP_WRITE)] = { 287 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 288 [C(RESULT_MISS)] = 0xd4, /* ITLB_MISS_INT */ 289 }, 290 [C(OP_PREFETCH)] = { 291 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 292 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 293 }, 294}, 295[C(BPU)] = { 296 [C(OP_READ)] = { 297 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 298 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 299 }, 300 [C(OP_WRITE)] = { 301 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 302 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 303 }, 304 [C(OP_PREFETCH)] = { 305 [C(RESULT_ACCESS)] = TILE_OP_UNSUPP, 306 [C(RESULT_MISS)] = TILE_OP_UNSUPP, 307 }, 308}, 309}; 310#endif 311 312static atomic_t tile_active_events; 313static DEFINE_MUTEX(perf_intr_reserve_mutex); 314 315static int tile_map_hw_event(u64 config); 316static int tile_map_cache_event(u64 config); 317 318static int tile_pmu_handle_irq(struct pt_regs *regs, int fault); 319 320/* 321 * To avoid new_raw_count getting larger then pre_raw_count 322 * in tile_perf_event_update(), we limit the value of max_period to 2^31 - 1. 323 */ 324static const struct tile_pmu tilepmu = { 325#ifndef __tilegx__ 326 .name = "tilepro", 327#else 328 .name = "tilegx", 329#endif 330 .max_events = ARRAY_SIZE(tile_hw_event_map), 331 .map_hw_event = tile_map_hw_event, 332 .hw_events = tile_hw_event_map, 333 .map_cache_event = tile_map_cache_event, 334 .cache_events = &tile_cache_event_map, 335 .cntval_bits = 32, 336 .cntval_mask = (1ULL << 32) - 1, 337 .max_period = (1ULL << 31) - 1, 338 .num_counters = TILE_MAX_COUNTERS, 339 .num_base_counters = TILE_BASE_COUNTERS, 340}; 341 342static const struct tile_pmu *tile_pmu __read_mostly; 343 344/* 345 * Check whether perf event is enabled. 346 */ 347int tile_perf_enabled(void) 348{ 349 return atomic_read(&tile_active_events) != 0; 350} 351 352/* 353 * Read Performance Counters. 354 */ 355static inline u64 read_counter(int idx) 356{ 357 u64 val = 0; 358 359 /* __insn_mfspr() only takes an immediate argument */ 360 switch (idx) { 361 case PERF_COUNT_0_IDX: 362 val = __insn_mfspr(SPR_PERF_COUNT_0); 363 break; 364 case PERF_COUNT_1_IDX: 365 val = __insn_mfspr(SPR_PERF_COUNT_1); 366 break; 367 case AUX_PERF_COUNT_0_IDX: 368 val = __insn_mfspr(SPR_AUX_PERF_COUNT_0); 369 break; 370 case AUX_PERF_COUNT_1_IDX: 371 val = __insn_mfspr(SPR_AUX_PERF_COUNT_1); 372 break; 373 default: 374 WARN_ON_ONCE(idx > AUX_PERF_COUNT_1_IDX || 375 idx < PERF_COUNT_0_IDX); 376 } 377 378 return val; 379} 380 381/* 382 * Write Performance Counters. 383 */ 384static inline void write_counter(int idx, u64 value) 385{ 386 /* __insn_mtspr() only takes an immediate argument */ 387 switch (idx) { 388 case PERF_COUNT_0_IDX: 389 __insn_mtspr(SPR_PERF_COUNT_0, value); 390 break; 391 case PERF_COUNT_1_IDX: 392 __insn_mtspr(SPR_PERF_COUNT_1, value); 393 break; 394 case AUX_PERF_COUNT_0_IDX: 395 __insn_mtspr(SPR_AUX_PERF_COUNT_0, value); 396 break; 397 case AUX_PERF_COUNT_1_IDX: 398 __insn_mtspr(SPR_AUX_PERF_COUNT_1, value); 399 break; 400 default: 401 WARN_ON_ONCE(idx > AUX_PERF_COUNT_1_IDX || 402 idx < PERF_COUNT_0_IDX); 403 } 404} 405 406/* 407 * Enable performance event by setting 408 * Performance Counter Control registers. 409 */ 410static inline void tile_pmu_enable_event(struct perf_event *event) 411{ 412 struct hw_perf_event *hwc = &event->hw; 413 unsigned long cfg, mask; 414 int shift, idx = hwc->idx; 415 416 /* 417 * prevent early activation from tile_pmu_start() in hw_perf_enable 418 */ 419 420 if (WARN_ON_ONCE(idx == -1)) 421 return; 422 423 if (idx < tile_pmu->num_base_counters) 424 cfg = __insn_mfspr(SPR_PERF_COUNT_CTL); 425 else 426 cfg = __insn_mfspr(SPR_AUX_PERF_COUNT_CTL); 427 428 switch (idx) { 429 case PERF_COUNT_0_IDX: 430 case AUX_PERF_COUNT_0_IDX: 431 mask = TILE_EVENT_MASK; 432 shift = 0; 433 break; 434 case PERF_COUNT_1_IDX: 435 case AUX_PERF_COUNT_1_IDX: 436 mask = TILE_EVENT_MASK << 16; 437 shift = 16; 438 break; 439 default: 440 WARN_ON_ONCE(idx < PERF_COUNT_0_IDX || 441 idx > AUX_PERF_COUNT_1_IDX); 442 return; 443 } 444 445 /* Clear mask bits to enable the event. */ 446 cfg &= ~mask; 447 cfg |= hwc->config << shift; 448 449 if (idx < tile_pmu->num_base_counters) 450 __insn_mtspr(SPR_PERF_COUNT_CTL, cfg); 451 else 452 __insn_mtspr(SPR_AUX_PERF_COUNT_CTL, cfg); 453} 454 455/* 456 * Disable performance event by clearing 457 * Performance Counter Control registers. 458 */ 459static inline void tile_pmu_disable_event(struct perf_event *event) 460{ 461 struct hw_perf_event *hwc = &event->hw; 462 unsigned long cfg, mask; 463 int idx = hwc->idx; 464 465 if (idx == -1) 466 return; 467 468 if (idx < tile_pmu->num_base_counters) 469 cfg = __insn_mfspr(SPR_PERF_COUNT_CTL); 470 else 471 cfg = __insn_mfspr(SPR_AUX_PERF_COUNT_CTL); 472 473 switch (idx) { 474 case PERF_COUNT_0_IDX: 475 case AUX_PERF_COUNT_0_IDX: 476 mask = TILE_PLM_MASK; 477 break; 478 case PERF_COUNT_1_IDX: 479 case AUX_PERF_COUNT_1_IDX: 480 mask = TILE_PLM_MASK << 16; 481 break; 482 default: 483 WARN_ON_ONCE(idx < PERF_COUNT_0_IDX || 484 idx > AUX_PERF_COUNT_1_IDX); 485 return; 486 } 487 488 /* Set mask bits to disable the event. */ 489 cfg |= mask; 490 491 if (idx < tile_pmu->num_base_counters) 492 __insn_mtspr(SPR_PERF_COUNT_CTL, cfg); 493 else 494 __insn_mtspr(SPR_AUX_PERF_COUNT_CTL, cfg); 495} 496 497/* 498 * Propagate event elapsed time into the generic event. 499 * Can only be executed on the CPU where the event is active. 500 * Returns the delta events processed. 501 */ 502static u64 tile_perf_event_update(struct perf_event *event) 503{ 504 struct hw_perf_event *hwc = &event->hw; 505 int shift = 64 - tile_pmu->cntval_bits; 506 u64 prev_raw_count, new_raw_count; 507 u64 oldval; 508 int idx = hwc->idx; 509 u64 delta; 510 511 /* 512 * Careful: an NMI might modify the previous event value. 513 * 514 * Our tactic to handle this is to first atomically read and 515 * exchange a new raw count - then add that new-prev delta 516 * count to the generic event atomically: 517 */ 518again: 519 prev_raw_count = local64_read(&hwc->prev_count); 520 new_raw_count = read_counter(idx); 521 522 oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count, 523 new_raw_count); 524 if (oldval != prev_raw_count) 525 goto again; 526 527 /* 528 * Now we have the new raw value and have updated the prev 529 * timestamp already. We can now calculate the elapsed delta 530 * (event-)time and add that to the generic event. 531 * 532 * Careful, not all hw sign-extends above the physical width 533 * of the count. 534 */ 535 delta = (new_raw_count << shift) - (prev_raw_count << shift); 536 delta >>= shift; 537 538 local64_add(delta, &event->count); 539 local64_sub(delta, &hwc->period_left); 540 541 return new_raw_count; 542} 543 544/* 545 * Set the next IRQ period, based on the hwc->period_left value. 546 * To be called with the event disabled in hw: 547 */ 548static int tile_event_set_period(struct perf_event *event) 549{ 550 struct hw_perf_event *hwc = &event->hw; 551 int idx = hwc->idx; 552 s64 left = local64_read(&hwc->period_left); 553 s64 period = hwc->sample_period; 554 int ret = 0; 555 556 /* 557 * If we are way outside a reasonable range then just skip forward: 558 */ 559 if (unlikely(left <= -period)) { 560 left = period; 561 local64_set(&hwc->period_left, left); 562 hwc->last_period = period; 563 ret = 1; 564 } 565 566 if (unlikely(left <= 0)) { 567 left += period; 568 local64_set(&hwc->period_left, left); 569 hwc->last_period = period; 570 ret = 1; 571 } 572 if (left > tile_pmu->max_period) 573 left = tile_pmu->max_period; 574 575 /* 576 * The hw event starts counting from this event offset, 577 * mark it to be able to extra future deltas: 578 */ 579 local64_set(&hwc->prev_count, (u64)-left); 580 581 write_counter(idx, (u64)(-left) & tile_pmu->cntval_mask); 582 583 perf_event_update_userpage(event); 584 585 return ret; 586} 587 588/* 589 * Stop the event but do not release the PMU counter 590 */ 591static void tile_pmu_stop(struct perf_event *event, int flags) 592{ 593 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 594 struct hw_perf_event *hwc = &event->hw; 595 int idx = hwc->idx; 596 597 if (__test_and_clear_bit(idx, cpuc->active_mask)) { 598 tile_pmu_disable_event(event); 599 cpuc->events[hwc->idx] = NULL; 600 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 601 hwc->state |= PERF_HES_STOPPED; 602 } 603 604 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 605 /* 606 * Drain the remaining delta count out of a event 607 * that we are disabling: 608 */ 609 tile_perf_event_update(event); 610 hwc->state |= PERF_HES_UPTODATE; 611 } 612} 613 614/* 615 * Start an event (without re-assigning counter) 616 */ 617static void tile_pmu_start(struct perf_event *event, int flags) 618{ 619 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 620 int idx = event->hw.idx; 621 622 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 623 return; 624 625 if (WARN_ON_ONCE(idx == -1)) 626 return; 627 628 if (flags & PERF_EF_RELOAD) { 629 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 630 tile_event_set_period(event); 631 } 632 633 event->hw.state = 0; 634 635 cpuc->events[idx] = event; 636 __set_bit(idx, cpuc->active_mask); 637 638 unmask_pmc_interrupts(); 639 640 tile_pmu_enable_event(event); 641 642 perf_event_update_userpage(event); 643} 644 645/* 646 * Add a single event to the PMU. 647 * 648 * The event is added to the group of enabled events 649 * but only if it can be scehduled with existing events. 650 */ 651static int tile_pmu_add(struct perf_event *event, int flags) 652{ 653 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 654 struct hw_perf_event *hwc; 655 unsigned long mask; 656 int b, max_cnt; 657 658 hwc = &event->hw; 659 660 /* 661 * We are full. 662 */ 663 if (cpuc->n_events == tile_pmu->num_counters) 664 return -ENOSPC; 665 666 cpuc->event_list[cpuc->n_events] = event; 667 cpuc->n_events++; 668 669 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 670 if (!(flags & PERF_EF_START)) 671 hwc->state |= PERF_HES_ARCH; 672 673 /* 674 * Find first empty counter. 675 */ 676 max_cnt = tile_pmu->num_counters; 677 mask = ~cpuc->used_mask; 678 679 /* Find next free counter. */ 680 b = find_next_bit(&mask, max_cnt, 0); 681 682 /* Should not happen. */ 683 if (WARN_ON_ONCE(b == max_cnt)) 684 return -ENOSPC; 685 686 /* 687 * Assign counter to event. 688 */ 689 event->hw.idx = b; 690 __set_bit(b, &cpuc->used_mask); 691 692 /* 693 * Start if requested. 694 */ 695 if (flags & PERF_EF_START) 696 tile_pmu_start(event, PERF_EF_RELOAD); 697 698 return 0; 699} 700 701/* 702 * Delete a single event from the PMU. 703 * 704 * The event is deleted from the group of enabled events. 705 * If it is the last event, disable PMU interrupt. 706 */ 707static void tile_pmu_del(struct perf_event *event, int flags) 708{ 709 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 710 int i; 711 712 /* 713 * Remove event from list, compact list if necessary. 714 */ 715 for (i = 0; i < cpuc->n_events; i++) { 716 if (cpuc->event_list[i] == event) { 717 while (++i < cpuc->n_events) 718 cpuc->event_list[i-1] = cpuc->event_list[i]; 719 --cpuc->n_events; 720 cpuc->events[event->hw.idx] = NULL; 721 __clear_bit(event->hw.idx, &cpuc->used_mask); 722 tile_pmu_stop(event, PERF_EF_UPDATE); 723 break; 724 } 725 } 726 /* 727 * If there are no events left, then mask PMU interrupt. 728 */ 729 if (cpuc->n_events == 0) 730 mask_pmc_interrupts(); 731 perf_event_update_userpage(event); 732} 733 734/* 735 * Propagate event elapsed time into the event. 736 */ 737static inline void tile_pmu_read(struct perf_event *event) 738{ 739 tile_perf_event_update(event); 740} 741 742/* 743 * Map generic events to Tile PMU. 744 */ 745static int tile_map_hw_event(u64 config) 746{ 747 if (config >= tile_pmu->max_events) 748 return -EINVAL; 749 return tile_pmu->hw_events[config]; 750} 751 752/* 753 * Map generic hardware cache events to Tile PMU. 754 */ 755static int tile_map_cache_event(u64 config) 756{ 757 unsigned int cache_type, cache_op, cache_result; 758 int code; 759 760 if (!tile_pmu->cache_events) 761 return -ENOENT; 762 763 cache_type = (config >> 0) & 0xff; 764 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 765 return -EINVAL; 766 767 cache_op = (config >> 8) & 0xff; 768 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 769 return -EINVAL; 770 771 cache_result = (config >> 16) & 0xff; 772 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 773 return -EINVAL; 774 775 code = (*tile_pmu->cache_events)[cache_type][cache_op][cache_result]; 776 if (code == TILE_OP_UNSUPP) 777 return -EINVAL; 778 779 return code; 780} 781 782static void tile_event_destroy(struct perf_event *event) 783{ 784 if (atomic_dec_return(&tile_active_events) == 0) 785 release_pmc_hardware(); 786} 787 788static int __tile_event_init(struct perf_event *event) 789{ 790 struct perf_event_attr *attr = &event->attr; 791 struct hw_perf_event *hwc = &event->hw; 792 int code; 793 794 switch (attr->type) { 795 case PERF_TYPE_HARDWARE: 796 code = tile_pmu->map_hw_event(attr->config); 797 break; 798 case PERF_TYPE_HW_CACHE: 799 code = tile_pmu->map_cache_event(attr->config); 800 break; 801 case PERF_TYPE_RAW: 802 code = attr->config & TILE_EVENT_MASK; 803 break; 804 default: 805 /* Should not happen. */ 806 return -EOPNOTSUPP; 807 } 808 809 if (code < 0) 810 return code; 811 812 hwc->config = code; 813 hwc->idx = -1; 814 815 if (attr->exclude_user) 816 hwc->config |= TILE_CTL_EXCL_USER; 817 818 if (attr->exclude_kernel) 819 hwc->config |= TILE_CTL_EXCL_KERNEL; 820 821 if (attr->exclude_hv) 822 hwc->config |= TILE_CTL_EXCL_HV; 823 824 if (!hwc->sample_period) { 825 hwc->sample_period = tile_pmu->max_period; 826 hwc->last_period = hwc->sample_period; 827 local64_set(&hwc->period_left, hwc->sample_period); 828 } 829 event->destroy = tile_event_destroy; 830 return 0; 831} 832 833static int tile_event_init(struct perf_event *event) 834{ 835 int err = 0; 836 perf_irq_t old_irq_handler = NULL; 837 838 if (atomic_inc_return(&tile_active_events) == 1) 839 old_irq_handler = reserve_pmc_hardware(tile_pmu_handle_irq); 840 841 if (old_irq_handler) { 842 pr_warn("PMC hardware busy (reserved by oprofile)\n"); 843 844 atomic_dec(&tile_active_events); 845 return -EBUSY; 846 } 847 848 switch (event->attr.type) { 849 case PERF_TYPE_RAW: 850 case PERF_TYPE_HARDWARE: 851 case PERF_TYPE_HW_CACHE: 852 break; 853 854 default: 855 return -ENOENT; 856 } 857 858 err = __tile_event_init(event); 859 if (err) { 860 if (event->destroy) 861 event->destroy(event); 862 } 863 return err; 864} 865 866static struct pmu tilera_pmu = { 867 .event_init = tile_event_init, 868 .add = tile_pmu_add, 869 .del = tile_pmu_del, 870 871 .start = tile_pmu_start, 872 .stop = tile_pmu_stop, 873 874 .read = tile_pmu_read, 875}; 876 877/* 878 * PMU's IRQ handler, PMU has 2 interrupts, they share the same handler. 879 */ 880int tile_pmu_handle_irq(struct pt_regs *regs, int fault) 881{ 882 struct perf_sample_data data; 883 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 884 struct perf_event *event; 885 struct hw_perf_event *hwc; 886 u64 val; 887 unsigned long status; 888 int bit; 889 890 __this_cpu_inc(perf_irqs); 891 892 if (!atomic_read(&tile_active_events)) 893 return 0; 894 895 status = pmc_get_overflow(); 896 pmc_ack_overflow(status); 897 898 for_each_set_bit(bit, &status, tile_pmu->num_counters) { 899 900 event = cpuc->events[bit]; 901 902 if (!event) 903 continue; 904 905 if (!test_bit(bit, cpuc->active_mask)) 906 continue; 907 908 hwc = &event->hw; 909 910 val = tile_perf_event_update(event); 911 if (val & (1ULL << (tile_pmu->cntval_bits - 1))) 912 continue; 913 914 perf_sample_data_init(&data, 0, event->hw.last_period); 915 if (!tile_event_set_period(event)) 916 continue; 917 918 if (perf_event_overflow(event, &data, regs)) 919 tile_pmu_stop(event, 0); 920 } 921 922 return 0; 923} 924 925static bool __init supported_pmu(void) 926{ 927 tile_pmu = &tilepmu; 928 return true; 929} 930 931int __init init_hw_perf_events(void) 932{ 933 supported_pmu(); 934 perf_pmu_register(&tilera_pmu, "cpu", PERF_TYPE_RAW); 935 return 0; 936} 937arch_initcall(init_hw_perf_events); 938 939/* Callchain handling code. */ 940 941/* 942 * Tile specific backtracing code for perf_events. 943 */ 944static inline void perf_callchain(struct perf_callchain_entry *entry, 945 struct pt_regs *regs) 946{ 947 struct KBacktraceIterator kbt; 948 unsigned int i; 949 950 /* 951 * Get the address just after the "jalr" instruction that 952 * jumps to the handler for a syscall. When we find this 953 * address in a backtrace, we silently ignore it, which gives 954 * us a one-step backtrace connection from the sys_xxx() 955 * function in the kernel to the xxx() function in libc. 956 * Otherwise, we lose the ability to properly attribute time 957 * from the libc calls to the kernel implementations, since 958 * oprofile only considers PCs from backtraces a pair at a time. 959 */ 960 unsigned long handle_syscall_pc = handle_syscall_link_address(); 961 962 KBacktraceIterator_init(&kbt, NULL, regs); 963 kbt.profile = 1; 964 965 /* 966 * The sample for the pc is already recorded. Now we are adding the 967 * address of the callsites on the stack. Our iterator starts 968 * with the frame of the (already sampled) call site. If our 969 * iterator contained a "return address" field, we could have just 970 * used it and wouldn't have needed to skip the first 971 * frame. That's in effect what the arm and x86 versions do. 972 * Instead we peel off the first iteration to get the equivalent 973 * behavior. 974 */ 975 976 if (KBacktraceIterator_end(&kbt)) 977 return; 978 KBacktraceIterator_next(&kbt); 979 980 /* 981 * Set stack depth to 16 for user and kernel space respectively, that 982 * is, total 32 stack frames. 983 */ 984 for (i = 0; i < 16; ++i) { 985 unsigned long pc; 986 if (KBacktraceIterator_end(&kbt)) 987 break; 988 pc = kbt.it.pc; 989 if (pc != handle_syscall_pc) 990 perf_callchain_store(entry, pc); 991 KBacktraceIterator_next(&kbt); 992 } 993} 994 995void perf_callchain_user(struct perf_callchain_entry *entry, 996 struct pt_regs *regs) 997{ 998 perf_callchain(entry, regs); 999} 1000 1001void perf_callchain_kernel(struct perf_callchain_entry *entry, 1002 struct pt_regs *regs) 1003{ 1004 perf_callchain(entry, regs); 1005} 1006