1 2#ifndef _LINUX_FTRACE_EVENT_H 3#define _LINUX_FTRACE_EVENT_H 4 5#include <linux/ring_buffer.h> 6#include <linux/trace_seq.h> 7#include <linux/percpu.h> 8#include <linux/hardirq.h> 9#include <linux/perf_event.h> 10#include <linux/tracepoint.h> 11 12struct trace_array; 13struct trace_buffer; 14struct tracer; 15struct dentry; 16struct bpf_prog; 17 18struct trace_print_flags { 19 unsigned long mask; 20 const char *name; 21}; 22 23struct trace_print_flags_u64 { 24 unsigned long long mask; 25 const char *name; 26}; 27 28const char *ftrace_print_flags_seq(struct trace_seq *p, const char *delim, 29 unsigned long flags, 30 const struct trace_print_flags *flag_array); 31 32const char *ftrace_print_symbols_seq(struct trace_seq *p, unsigned long val, 33 const struct trace_print_flags *symbol_array); 34 35#if BITS_PER_LONG == 32 36const char *ftrace_print_symbols_seq_u64(struct trace_seq *p, 37 unsigned long long val, 38 const struct trace_print_flags_u64 39 *symbol_array); 40#endif 41 42const char *ftrace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr, 43 unsigned int bitmask_size); 44 45const char *ftrace_print_hex_seq(struct trace_seq *p, 46 const unsigned char *buf, int len); 47 48const char *ftrace_print_array_seq(struct trace_seq *p, 49 const void *buf, int count, 50 size_t el_size); 51 52struct trace_iterator; 53struct trace_event; 54 55int ftrace_raw_output_prep(struct trace_iterator *iter, 56 struct trace_event *event); 57 58/* 59 * The trace entry - the most basic unit of tracing. This is what 60 * is printed in the end as a single line in the trace output, such as: 61 * 62 * bash-15816 [01] 235.197585: idle_cpu <- irq_enter 63 */ 64struct trace_entry { 65 unsigned short type; 66 unsigned char flags; 67 unsigned char preempt_count; 68 int pid; 69}; 70 71#define FTRACE_MAX_EVENT \ 72 ((1 << (sizeof(((struct trace_entry *)0)->type) * 8)) - 1) 73 74/* 75 * Trace iterator - used by printout routines who present trace 76 * results to users and which routines might sleep, etc: 77 */ 78struct trace_iterator { 79 struct trace_array *tr; 80 struct tracer *trace; 81 struct trace_buffer *trace_buffer; 82 void *private; 83 int cpu_file; 84 struct mutex mutex; 85 struct ring_buffer_iter **buffer_iter; 86 unsigned long iter_flags; 87 88 /* trace_seq for __print_flags() and __print_symbolic() etc. */ 89 struct trace_seq tmp_seq; 90 91 cpumask_var_t started; 92 93 /* it's true when current open file is snapshot */ 94 bool snapshot; 95 96 /* The below is zeroed out in pipe_read */ 97 struct trace_seq seq; 98 struct trace_entry *ent; 99 unsigned long lost_events; 100 int leftover; 101 int ent_size; 102 int cpu; 103 u64 ts; 104 105 loff_t pos; 106 long idx; 107 108 /* All new field here will be zeroed out in pipe_read */ 109}; 110 111enum trace_iter_flags { 112 TRACE_FILE_LAT_FMT = 1, 113 TRACE_FILE_ANNOTATE = 2, 114 TRACE_FILE_TIME_IN_NS = 4, 115}; 116 117 118typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter, 119 int flags, struct trace_event *event); 120 121struct trace_event_functions { 122 trace_print_func trace; 123 trace_print_func raw; 124 trace_print_func hex; 125 trace_print_func binary; 126}; 127 128struct trace_event { 129 struct hlist_node node; 130 struct list_head list; 131 int type; 132 struct trace_event_functions *funcs; 133}; 134 135extern int register_ftrace_event(struct trace_event *event); 136extern int unregister_ftrace_event(struct trace_event *event); 137 138/* Return values for print_line callback */ 139enum print_line_t { 140 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */ 141 TRACE_TYPE_HANDLED = 1, 142 TRACE_TYPE_UNHANDLED = 2, /* Relay to other output functions */ 143 TRACE_TYPE_NO_CONSUME = 3 /* Handled but ask to not consume */ 144}; 145 146/* 147 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq 148 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function 149 * simplifies those functions and keeps them in sync. 150 */ 151static inline enum print_line_t trace_handle_return(struct trace_seq *s) 152{ 153 return trace_seq_has_overflowed(s) ? 154 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; 155} 156 157void tracing_generic_entry_update(struct trace_entry *entry, 158 unsigned long flags, 159 int pc); 160struct ftrace_event_file; 161 162struct ring_buffer_event * 163trace_event_buffer_lock_reserve(struct ring_buffer **current_buffer, 164 struct ftrace_event_file *ftrace_file, 165 int type, unsigned long len, 166 unsigned long flags, int pc); 167struct ring_buffer_event * 168trace_current_buffer_lock_reserve(struct ring_buffer **current_buffer, 169 int type, unsigned long len, 170 unsigned long flags, int pc); 171void trace_current_buffer_unlock_commit(struct ring_buffer *buffer, 172 struct ring_buffer_event *event, 173 unsigned long flags, int pc); 174void trace_buffer_unlock_commit(struct ring_buffer *buffer, 175 struct ring_buffer_event *event, 176 unsigned long flags, int pc); 177void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer, 178 struct ring_buffer_event *event, 179 unsigned long flags, int pc, 180 struct pt_regs *regs); 181void trace_current_buffer_discard_commit(struct ring_buffer *buffer, 182 struct ring_buffer_event *event); 183 184void tracing_record_cmdline(struct task_struct *tsk); 185 186int ftrace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...); 187 188struct event_filter; 189 190enum trace_reg { 191 TRACE_REG_REGISTER, 192 TRACE_REG_UNREGISTER, 193#ifdef CONFIG_PERF_EVENTS 194 TRACE_REG_PERF_REGISTER, 195 TRACE_REG_PERF_UNREGISTER, 196 TRACE_REG_PERF_OPEN, 197 TRACE_REG_PERF_CLOSE, 198 TRACE_REG_PERF_ADD, 199 TRACE_REG_PERF_DEL, 200#endif 201}; 202 203struct ftrace_event_call; 204 205struct ftrace_event_class { 206 const char *system; 207 void *probe; 208#ifdef CONFIG_PERF_EVENTS 209 void *perf_probe; 210#endif 211 int (*reg)(struct ftrace_event_call *event, 212 enum trace_reg type, void *data); 213 int (*define_fields)(struct ftrace_event_call *); 214 struct list_head *(*get_fields)(struct ftrace_event_call *); 215 struct list_head fields; 216 int (*raw_init)(struct ftrace_event_call *); 217}; 218 219extern int ftrace_event_reg(struct ftrace_event_call *event, 220 enum trace_reg type, void *data); 221 222int ftrace_output_event(struct trace_iterator *iter, struct ftrace_event_call *event, 223 char *fmt, ...); 224 225int ftrace_event_define_field(struct ftrace_event_call *call, 226 char *type, int len, char *item, int offset, 227 int field_size, int sign, int filter); 228 229struct ftrace_event_buffer { 230 struct ring_buffer *buffer; 231 struct ring_buffer_event *event; 232 struct ftrace_event_file *ftrace_file; 233 void *entry; 234 unsigned long flags; 235 int pc; 236}; 237 238void *ftrace_event_buffer_reserve(struct ftrace_event_buffer *fbuffer, 239 struct ftrace_event_file *ftrace_file, 240 unsigned long len); 241 242void ftrace_event_buffer_commit(struct ftrace_event_buffer *fbuffer); 243 244int ftrace_event_define_field(struct ftrace_event_call *call, 245 char *type, int len, char *item, int offset, 246 int field_size, int sign, int filter); 247 248enum { 249 TRACE_EVENT_FL_FILTERED_BIT, 250 TRACE_EVENT_FL_CAP_ANY_BIT, 251 TRACE_EVENT_FL_NO_SET_FILTER_BIT, 252 TRACE_EVENT_FL_IGNORE_ENABLE_BIT, 253 TRACE_EVENT_FL_WAS_ENABLED_BIT, 254 TRACE_EVENT_FL_USE_CALL_FILTER_BIT, 255 TRACE_EVENT_FL_TRACEPOINT_BIT, 256 TRACE_EVENT_FL_KPROBE_BIT, 257}; 258 259/* 260 * Event flags: 261 * FILTERED - The event has a filter attached 262 * CAP_ANY - Any user can enable for perf 263 * NO_SET_FILTER - Set when filter has error and is to be ignored 264 * IGNORE_ENABLE - For ftrace internal events, do not enable with debugfs file 265 * WAS_ENABLED - Set and stays set when an event was ever enabled 266 * (used for module unloading, if a module event is enabled, 267 * it is best to clear the buffers that used it). 268 * USE_CALL_FILTER - For ftrace internal events, don't use file filter 269 * TRACEPOINT - Event is a tracepoint 270 * KPROBE - Event is a kprobe 271 */ 272enum { 273 TRACE_EVENT_FL_FILTERED = (1 << TRACE_EVENT_FL_FILTERED_BIT), 274 TRACE_EVENT_FL_CAP_ANY = (1 << TRACE_EVENT_FL_CAP_ANY_BIT), 275 TRACE_EVENT_FL_NO_SET_FILTER = (1 << TRACE_EVENT_FL_NO_SET_FILTER_BIT), 276 TRACE_EVENT_FL_IGNORE_ENABLE = (1 << TRACE_EVENT_FL_IGNORE_ENABLE_BIT), 277 TRACE_EVENT_FL_WAS_ENABLED = (1 << TRACE_EVENT_FL_WAS_ENABLED_BIT), 278 TRACE_EVENT_FL_USE_CALL_FILTER = (1 << TRACE_EVENT_FL_USE_CALL_FILTER_BIT), 279 TRACE_EVENT_FL_TRACEPOINT = (1 << TRACE_EVENT_FL_TRACEPOINT_BIT), 280 TRACE_EVENT_FL_KPROBE = (1 << TRACE_EVENT_FL_KPROBE_BIT), 281}; 282 283struct ftrace_event_call { 284 struct list_head list; 285 struct ftrace_event_class *class; 286 union { 287 char *name; 288 /* Set TRACE_EVENT_FL_TRACEPOINT flag when using "tp" */ 289 struct tracepoint *tp; 290 }; 291 struct trace_event event; 292 char *print_fmt; 293 struct event_filter *filter; 294 void *mod; 295 void *data; 296 /* 297 * bit 0: filter_active 298 * bit 1: allow trace by non root (cap any) 299 * bit 2: failed to apply filter 300 * bit 3: ftrace internal event (do not enable) 301 * bit 4: Event was enabled by module 302 * bit 5: use call filter rather than file filter 303 * bit 6: Event is a tracepoint 304 */ 305 int flags; /* static flags of different events */ 306 307#ifdef CONFIG_PERF_EVENTS 308 int perf_refcount; 309 struct hlist_head __percpu *perf_events; 310 struct bpf_prog *prog; 311 312 int (*perf_perm)(struct ftrace_event_call *, 313 struct perf_event *); 314#endif 315}; 316 317static inline const char * 318ftrace_event_name(struct ftrace_event_call *call) 319{ 320 if (call->flags & TRACE_EVENT_FL_TRACEPOINT) 321 return call->tp ? call->tp->name : NULL; 322 else 323 return call->name; 324} 325 326struct trace_array; 327struct ftrace_subsystem_dir; 328 329enum { 330 FTRACE_EVENT_FL_ENABLED_BIT, 331 FTRACE_EVENT_FL_RECORDED_CMD_BIT, 332 FTRACE_EVENT_FL_FILTERED_BIT, 333 FTRACE_EVENT_FL_NO_SET_FILTER_BIT, 334 FTRACE_EVENT_FL_SOFT_MODE_BIT, 335 FTRACE_EVENT_FL_SOFT_DISABLED_BIT, 336 FTRACE_EVENT_FL_TRIGGER_MODE_BIT, 337 FTRACE_EVENT_FL_TRIGGER_COND_BIT, 338}; 339 340/* 341 * Ftrace event file flags: 342 * ENABLED - The event is enabled 343 * RECORDED_CMD - The comms should be recorded at sched_switch 344 * FILTERED - The event has a filter attached 345 * NO_SET_FILTER - Set when filter has error and is to be ignored 346 * SOFT_MODE - The event is enabled/disabled by SOFT_DISABLED 347 * SOFT_DISABLED - When set, do not trace the event (even though its 348 * tracepoint may be enabled) 349 * TRIGGER_MODE - When set, invoke the triggers associated with the event 350 * TRIGGER_COND - When set, one or more triggers has an associated filter 351 */ 352enum { 353 FTRACE_EVENT_FL_ENABLED = (1 << FTRACE_EVENT_FL_ENABLED_BIT), 354 FTRACE_EVENT_FL_RECORDED_CMD = (1 << FTRACE_EVENT_FL_RECORDED_CMD_BIT), 355 FTRACE_EVENT_FL_FILTERED = (1 << FTRACE_EVENT_FL_FILTERED_BIT), 356 FTRACE_EVENT_FL_NO_SET_FILTER = (1 << FTRACE_EVENT_FL_NO_SET_FILTER_BIT), 357 FTRACE_EVENT_FL_SOFT_MODE = (1 << FTRACE_EVENT_FL_SOFT_MODE_BIT), 358 FTRACE_EVENT_FL_SOFT_DISABLED = (1 << FTRACE_EVENT_FL_SOFT_DISABLED_BIT), 359 FTRACE_EVENT_FL_TRIGGER_MODE = (1 << FTRACE_EVENT_FL_TRIGGER_MODE_BIT), 360 FTRACE_EVENT_FL_TRIGGER_COND = (1 << FTRACE_EVENT_FL_TRIGGER_COND_BIT), 361}; 362 363struct ftrace_event_file { 364 struct list_head list; 365 struct ftrace_event_call *event_call; 366 struct event_filter *filter; 367 struct dentry *dir; 368 struct trace_array *tr; 369 struct ftrace_subsystem_dir *system; 370 struct list_head triggers; 371 372 /* 373 * 32 bit flags: 374 * bit 0: enabled 375 * bit 1: enabled cmd record 376 * bit 2: enable/disable with the soft disable bit 377 * bit 3: soft disabled 378 * bit 4: trigger enabled 379 * 380 * Note: The bits must be set atomically to prevent races 381 * from other writers. Reads of flags do not need to be in 382 * sync as they occur in critical sections. But the way flags 383 * is currently used, these changes do not affect the code 384 * except that when a change is made, it may have a slight 385 * delay in propagating the changes to other CPUs due to 386 * caching and such. Which is mostly OK ;-) 387 */ 388 unsigned long flags; 389 atomic_t sm_ref; /* soft-mode reference counter */ 390 atomic_t tm_ref; /* trigger-mode reference counter */ 391}; 392 393#define __TRACE_EVENT_FLAGS(name, value) \ 394 static int __init trace_init_flags_##name(void) \ 395 { \ 396 event_##name.flags |= value; \ 397 return 0; \ 398 } \ 399 early_initcall(trace_init_flags_##name); 400 401#define __TRACE_EVENT_PERF_PERM(name, expr...) \ 402 static int perf_perm_##name(struct ftrace_event_call *tp_event, \ 403 struct perf_event *p_event) \ 404 { \ 405 return ({ expr; }); \ 406 } \ 407 static int __init trace_init_perf_perm_##name(void) \ 408 { \ 409 event_##name.perf_perm = &perf_perm_##name; \ 410 return 0; \ 411 } \ 412 early_initcall(trace_init_perf_perm_##name); 413 414#define PERF_MAX_TRACE_SIZE 2048 415 416#define MAX_FILTER_STR_VAL 256 /* Should handle KSYM_SYMBOL_LEN */ 417 418enum event_trigger_type { 419 ETT_NONE = (0), 420 ETT_TRACE_ONOFF = (1 << 0), 421 ETT_SNAPSHOT = (1 << 1), 422 ETT_STACKTRACE = (1 << 2), 423 ETT_EVENT_ENABLE = (1 << 3), 424}; 425 426extern int filter_match_preds(struct event_filter *filter, void *rec); 427 428extern int filter_check_discard(struct ftrace_event_file *file, void *rec, 429 struct ring_buffer *buffer, 430 struct ring_buffer_event *event); 431extern int call_filter_check_discard(struct ftrace_event_call *call, void *rec, 432 struct ring_buffer *buffer, 433 struct ring_buffer_event *event); 434extern enum event_trigger_type event_triggers_call(struct ftrace_event_file *file, 435 void *rec); 436extern void event_triggers_post_call(struct ftrace_event_file *file, 437 enum event_trigger_type tt); 438 439/** 440 * ftrace_trigger_soft_disabled - do triggers and test if soft disabled 441 * @file: The file pointer of the event to test 442 * 443 * If any triggers without filters are attached to this event, they 444 * will be called here. If the event is soft disabled and has no 445 * triggers that require testing the fields, it will return true, 446 * otherwise false. 447 */ 448static inline bool 449ftrace_trigger_soft_disabled(struct ftrace_event_file *file) 450{ 451 unsigned long eflags = file->flags; 452 453 if (!(eflags & FTRACE_EVENT_FL_TRIGGER_COND)) { 454 if (eflags & FTRACE_EVENT_FL_TRIGGER_MODE) 455 event_triggers_call(file, NULL); 456 if (eflags & FTRACE_EVENT_FL_SOFT_DISABLED) 457 return true; 458 } 459 return false; 460} 461 462/* 463 * Helper function for event_trigger_unlock_commit{_regs}(). 464 * If there are event triggers attached to this event that requires 465 * filtering against its fields, then they wil be called as the 466 * entry already holds the field information of the current event. 467 * 468 * It also checks if the event should be discarded or not. 469 * It is to be discarded if the event is soft disabled and the 470 * event was only recorded to process triggers, or if the event 471 * filter is active and this event did not match the filters. 472 * 473 * Returns true if the event is discarded, false otherwise. 474 */ 475static inline bool 476__event_trigger_test_discard(struct ftrace_event_file *file, 477 struct ring_buffer *buffer, 478 struct ring_buffer_event *event, 479 void *entry, 480 enum event_trigger_type *tt) 481{ 482 unsigned long eflags = file->flags; 483 484 if (eflags & FTRACE_EVENT_FL_TRIGGER_COND) 485 *tt = event_triggers_call(file, entry); 486 487 if (test_bit(FTRACE_EVENT_FL_SOFT_DISABLED_BIT, &file->flags)) 488 ring_buffer_discard_commit(buffer, event); 489 else if (!filter_check_discard(file, entry, buffer, event)) 490 return false; 491 492 return true; 493} 494 495/** 496 * event_trigger_unlock_commit - handle triggers and finish event commit 497 * @file: The file pointer assoctiated to the event 498 * @buffer: The ring buffer that the event is being written to 499 * @event: The event meta data in the ring buffer 500 * @entry: The event itself 501 * @irq_flags: The state of the interrupts at the start of the event 502 * @pc: The state of the preempt count at the start of the event. 503 * 504 * This is a helper function to handle triggers that require data 505 * from the event itself. It also tests the event against filters and 506 * if the event is soft disabled and should be discarded. 507 */ 508static inline void 509event_trigger_unlock_commit(struct ftrace_event_file *file, 510 struct ring_buffer *buffer, 511 struct ring_buffer_event *event, 512 void *entry, unsigned long irq_flags, int pc) 513{ 514 enum event_trigger_type tt = ETT_NONE; 515 516 if (!__event_trigger_test_discard(file, buffer, event, entry, &tt)) 517 trace_buffer_unlock_commit(buffer, event, irq_flags, pc); 518 519 if (tt) 520 event_triggers_post_call(file, tt); 521} 522 523/** 524 * event_trigger_unlock_commit_regs - handle triggers and finish event commit 525 * @file: The file pointer assoctiated to the event 526 * @buffer: The ring buffer that the event is being written to 527 * @event: The event meta data in the ring buffer 528 * @entry: The event itself 529 * @irq_flags: The state of the interrupts at the start of the event 530 * @pc: The state of the preempt count at the start of the event. 531 * 532 * This is a helper function to handle triggers that require data 533 * from the event itself. It also tests the event against filters and 534 * if the event is soft disabled and should be discarded. 535 * 536 * Same as event_trigger_unlock_commit() but calls 537 * trace_buffer_unlock_commit_regs() instead of trace_buffer_unlock_commit(). 538 */ 539static inline void 540event_trigger_unlock_commit_regs(struct ftrace_event_file *file, 541 struct ring_buffer *buffer, 542 struct ring_buffer_event *event, 543 void *entry, unsigned long irq_flags, int pc, 544 struct pt_regs *regs) 545{ 546 enum event_trigger_type tt = ETT_NONE; 547 548 if (!__event_trigger_test_discard(file, buffer, event, entry, &tt)) 549 trace_buffer_unlock_commit_regs(buffer, event, 550 irq_flags, pc, regs); 551 552 if (tt) 553 event_triggers_post_call(file, tt); 554} 555 556#ifdef CONFIG_BPF_SYSCALL 557unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx); 558#else 559static inline unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx) 560{ 561 return 1; 562} 563#endif 564 565enum { 566 FILTER_OTHER = 0, 567 FILTER_STATIC_STRING, 568 FILTER_DYN_STRING, 569 FILTER_PTR_STRING, 570 FILTER_TRACE_FN, 571}; 572 573extern int trace_event_raw_init(struct ftrace_event_call *call); 574extern int trace_define_field(struct ftrace_event_call *call, const char *type, 575 const char *name, int offset, int size, 576 int is_signed, int filter_type); 577extern int trace_add_event_call(struct ftrace_event_call *call); 578extern int trace_remove_event_call(struct ftrace_event_call *call); 579 580#define is_signed_type(type) (((type)(-1)) < (type)1) 581 582int trace_set_clr_event(const char *system, const char *event, int set); 583 584/* 585 * The double __builtin_constant_p is because gcc will give us an error 586 * if we try to allocate the static variable to fmt if it is not a 587 * constant. Even with the outer if statement optimizing out. 588 */ 589#define event_trace_printk(ip, fmt, args...) \ 590do { \ 591 __trace_printk_check_format(fmt, ##args); \ 592 tracing_record_cmdline(current); \ 593 if (__builtin_constant_p(fmt)) { \ 594 static const char *trace_printk_fmt \ 595 __attribute__((section("__trace_printk_fmt"))) = \ 596 __builtin_constant_p(fmt) ? fmt : NULL; \ 597 \ 598 __trace_bprintk(ip, trace_printk_fmt, ##args); \ 599 } else \ 600 __trace_printk(ip, fmt, ##args); \ 601} while (0) 602 603#ifdef CONFIG_PERF_EVENTS 604struct perf_event; 605 606DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); 607 608extern int perf_trace_init(struct perf_event *event); 609extern void perf_trace_destroy(struct perf_event *event); 610extern int perf_trace_add(struct perf_event *event, int flags); 611extern void perf_trace_del(struct perf_event *event, int flags); 612extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, 613 char *filter_str); 614extern void ftrace_profile_free_filter(struct perf_event *event); 615extern void *perf_trace_buf_prepare(int size, unsigned short type, 616 struct pt_regs **regs, int *rctxp); 617 618static inline void 619perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr, 620 u64 count, struct pt_regs *regs, void *head, 621 struct task_struct *task) 622{ 623 perf_tp_event(addr, count, raw_data, size, regs, head, rctx, task); 624} 625#endif 626 627#endif /* _LINUX_FTRACE_EVENT_H */ 628