Lines Matching refs:rec
544 struct ftrace_profile *rec = v; in function_stat_next() local
547 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); in function_stat_next()
551 rec++; in function_stat_next()
553 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
557 rec = &pg->records[0]; in function_stat_next()
558 if (!rec->counter) in function_stat_next()
562 return rec; in function_stat_next()
622 struct ftrace_profile *rec = v; in function_stat_show() local
633 if (unlikely(rec->counter == 0)) { in function_stat_show()
639 avg = rec->time; in function_stat_show()
640 do_div(avg, rec->counter); in function_stat_show()
645 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); in function_stat_show()
646 seq_printf(m, " %-30.30s %10lu", str, rec->counter); in function_stat_show()
652 if (rec->counter <= 1) in function_stat_show()
659 stddev = rec->counter * rec->time_squared - in function_stat_show()
660 rec->time * rec->time; in function_stat_show()
666 do_div(stddev, rec->counter * (rec->counter - 1) * 1000); in function_stat_show()
670 trace_print_graph_duration(rec->time, &s); in function_stat_show()
808 struct ftrace_profile *rec; in ftrace_find_profiled_func() local
818 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { in ftrace_find_profiled_func()
819 if (rec->ip == ip) in ftrace_find_profiled_func()
820 return rec; in ftrace_find_profiled_func()
827 struct ftrace_profile *rec) in ftrace_add_profile() argument
831 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); in ftrace_add_profile()
832 hlist_add_head_rcu(&rec->node, &stat->hash[key]); in ftrace_add_profile()
841 struct ftrace_profile *rec = NULL; in ftrace_profile_alloc() local
851 rec = ftrace_find_profiled_func(stat, ip); in ftrace_profile_alloc()
852 if (rec) in ftrace_profile_alloc()
861 rec = &stat->pages->records[stat->pages->index++]; in ftrace_profile_alloc()
862 rec->ip = ip; in ftrace_profile_alloc()
863 ftrace_add_profile(stat, rec); in ftrace_profile_alloc()
868 return rec; in ftrace_profile_alloc()
876 struct ftrace_profile *rec; in function_profile_call() local
888 rec = ftrace_find_profiled_func(stat, ip); in function_profile_call()
889 if (!rec) { in function_profile_call()
890 rec = ftrace_profile_alloc(stat, ip); in function_profile_call()
891 if (!rec) in function_profile_call()
895 rec->counter++; in function_profile_call()
911 struct ftrace_profile *rec; in profile_graph_return() local
940 rec = ftrace_find_profiled_func(stat, trace->func); in profile_graph_return()
941 if (rec) { in profile_graph_return()
942 rec->time += calltime; in profile_graph_return()
943 rec->time_squared += calltime * calltime; in profile_graph_return()
1542 #define do_for_each_ftrace_rec(pg, rec) \ argument
1546 rec = &pg->records[_____i];
1556 const struct dyn_ftrace *rec = b; in ftrace_cmp_recs() local
1558 if (key->flags < rec->ip) in ftrace_cmp_recs()
1560 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) in ftrace_cmp_recs()
1568 struct dyn_ftrace *rec; in ftrace_location_range() local
1578 rec = bsearch(&key, pg->records, pg->index, in ftrace_location_range()
1581 if (rec) in ftrace_location_range()
1582 return rec->ip; in ftrace_location_range()
1623 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) in test_rec_ops_needs_regs() argument
1631 if (ftrace_ops_test(ops, rec->ip, rec)) { in test_rec_ops_needs_regs()
1649 struct dyn_ftrace *rec; in __ftrace_hash_rec_update() local
1685 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_rec_update()
1695 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip)) in __ftrace_hash_rec_update()
1698 in_hash = !!ftrace_lookup_ip(hash, rec->ip); in __ftrace_hash_rec_update()
1699 in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip); in __ftrace_hash_rec_update()
1721 rec->flags++; in __ftrace_hash_rec_update()
1722 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) in __ftrace_hash_rec_update()
1730 if (ftrace_rec_count(rec) == 1 && ops->trampoline) in __ftrace_hash_rec_update()
1731 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1739 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1746 rec->flags |= FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1748 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) in __ftrace_hash_rec_update()
1750 rec->flags--; in __ftrace_hash_rec_update()
1758 if (ftrace_rec_count(rec) > 0 && in __ftrace_hash_rec_update()
1759 rec->flags & FTRACE_FL_REGS && in __ftrace_hash_rec_update()
1761 if (!test_rec_ops_needs_regs(rec)) in __ftrace_hash_rec_update()
1762 rec->flags &= ~FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1774 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1849 struct dyn_ftrace *rec, *end = NULL; in __ftrace_hash_update_ipmodify() local
1867 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
1869 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1870 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1876 if (rec->flags & FTRACE_FL_IPMODIFY) in __ftrace_hash_update_ipmodify()
1878 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1880 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1886 end = rec; in __ftrace_hash_update_ipmodify()
1889 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
1890 if (rec == end) in __ftrace_hash_update_ipmodify()
1893 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1894 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
1899 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1901 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
1954 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1968 void ftrace_bug(int failed, struct dyn_ftrace *rec) in ftrace_bug() argument
1970 unsigned long ip = rec ? rec->ip : 0; in ftrace_bug()
1995 if (rec) { in ftrace_bug()
1998 pr_info("ftrace record flags: %lx\n", rec->flags); in ftrace_bug()
1999 pr_cont(" (%ld)%s", ftrace_rec_count(rec), in ftrace_bug()
2000 rec->flags & FTRACE_FL_REGS ? " R" : " "); in ftrace_bug()
2001 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_bug()
2002 ops = ftrace_find_tramp_ops_any(rec); in ftrace_bug()
2010 ip = ftrace_get_addr_curr(rec); in ftrace_bug()
2015 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) in ftrace_check_record() argument
2030 if (enable && ftrace_rec_count(rec)) in ftrace_check_record()
2039 if (!(rec->flags & FTRACE_FL_REGS) != in ftrace_check_record()
2040 !(rec->flags & FTRACE_FL_REGS_EN)) in ftrace_check_record()
2043 if (!(rec->flags & FTRACE_FL_TRAMP) != in ftrace_check_record()
2044 !(rec->flags & FTRACE_FL_TRAMP_EN)) in ftrace_check_record()
2049 if ((rec->flags & FTRACE_FL_ENABLED) == flag) in ftrace_check_record()
2054 flag ^= rec->flags & FTRACE_FL_ENABLED; in ftrace_check_record()
2057 rec->flags |= FTRACE_FL_ENABLED; in ftrace_check_record()
2059 if (rec->flags & FTRACE_FL_REGS) in ftrace_check_record()
2060 rec->flags |= FTRACE_FL_REGS_EN; in ftrace_check_record()
2062 rec->flags &= ~FTRACE_FL_REGS_EN; in ftrace_check_record()
2065 if (rec->flags & FTRACE_FL_TRAMP) in ftrace_check_record()
2066 rec->flags |= FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2068 rec->flags &= ~FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2088 if (!ftrace_rec_count(rec)) in ftrace_check_record()
2089 rec->flags = 0; in ftrace_check_record()
2095 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | in ftrace_check_record()
2110 int ftrace_update_record(struct dyn_ftrace *rec, int enable) in ftrace_update_record() argument
2112 return ftrace_check_record(rec, enable, 1); in ftrace_update_record()
2124 int ftrace_test_record(struct dyn_ftrace *rec, int enable) in ftrace_test_record() argument
2126 return ftrace_check_record(rec, enable, 0); in ftrace_test_record()
2130 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) in ftrace_find_tramp_ops_any() argument
2133 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any()
2148 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) in ftrace_find_tramp_ops_curr() argument
2151 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_curr()
2218 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) in ftrace_find_tramp_ops_new() argument
2221 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_new()
2242 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) in ftrace_get_addr_new() argument
2247 if (rec->flags & FTRACE_FL_TRAMP) { in ftrace_get_addr_new()
2248 ops = ftrace_find_tramp_ops_new(rec); in ftrace_get_addr_new()
2251 (void *)rec->ip, (void *)rec->ip, rec->flags); in ftrace_get_addr_new()
2258 if (rec->flags & FTRACE_FL_REGS) in ftrace_get_addr_new()
2274 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) in ftrace_get_addr_curr() argument
2279 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_get_addr_curr()
2280 ops = ftrace_find_tramp_ops_curr(rec); in ftrace_get_addr_curr()
2283 (void *)rec->ip, (void *)rec->ip); in ftrace_get_addr_curr()
2290 if (rec->flags & FTRACE_FL_REGS_EN) in ftrace_get_addr_curr()
2297 __ftrace_replace_code(struct dyn_ftrace *rec, int enable) in __ftrace_replace_code() argument
2303 ftrace_addr = ftrace_get_addr_new(rec); in __ftrace_replace_code()
2306 ftrace_old_addr = ftrace_get_addr_curr(rec); in __ftrace_replace_code()
2308 ret = ftrace_update_record(rec, enable); in __ftrace_replace_code()
2315 return ftrace_make_call(rec, ftrace_addr); in __ftrace_replace_code()
2318 return ftrace_make_nop(NULL, rec, ftrace_old_addr); in __ftrace_replace_code()
2321 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); in __ftrace_replace_code()
2329 struct dyn_ftrace *rec; in ftrace_replace_code() local
2336 do_for_each_ftrace_rec(pg, rec) { in ftrace_replace_code()
2337 failed = __ftrace_replace_code(rec, enable); in ftrace_replace_code()
2339 ftrace_bug(failed, rec); in ftrace_replace_code()
2419 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) in ftrace_code_disable() argument
2426 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); in ftrace_code_disable()
2428 ftrace_bug(ret, rec); in ftrace_code_disable()
2699 struct dyn_ftrace *rec; in ftrace_shutdown() local
2701 do_for_each_ftrace_rec(pg, rec) { in ftrace_shutdown()
2702 if (FTRACE_WARN_ON_ONCE(rec->flags)) in ftrace_shutdown()
2704 (void *)rec->ip, rec->flags); in ftrace_shutdown()
2795 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) in ops_references_rec() argument
2807 !ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip)) in ops_references_rec()
2811 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) in ops_references_rec()
2817 static int referenced_filters(struct dyn_ftrace *rec) in referenced_filters() argument
2823 if (ops_references_rec(ops, rec)) in referenced_filters()
3085 struct ftrace_func_probe *rec; in t_hash_show() local
3087 rec = iter->probe; in t_hash_show()
3088 if (WARN_ON_ONCE(!rec)) in t_hash_show()
3091 if (rec->ops->print) in t_hash_show()
3092 return rec->ops->print(m, rec->ip, rec->ops, rec->data); in t_hash_show()
3094 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); in t_hash_show()
3096 if (rec->data) in t_hash_show()
3097 seq_printf(m, ":%p", rec->data); in t_hash_show()
3108 struct dyn_ftrace *rec = NULL; in t_next() local
3130 rec = &iter->pg->records[iter->idx++]; in t_next()
3132 !(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) || in t_next()
3135 !ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) || in t_next()
3138 !(rec->flags & FTRACE_FL_ENABLED))) { in t_next()
3140 rec = NULL; in t_next()
3145 if (!rec) in t_next()
3148 iter->func = rec; in t_next()
3223 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) in arch_ftrace_trampoline_func() argument
3229 struct dyn_ftrace *rec) in add_trampoline_func() argument
3233 ptr = arch_ftrace_trampoline_func(ops, rec); in add_trampoline_func()
3241 struct dyn_ftrace *rec; in t_show() local
3254 rec = iter->func; in t_show()
3256 if (!rec) in t_show()
3259 seq_printf(m, "%ps", (void *)rec->ip); in t_show()
3264 ftrace_rec_count(rec), in t_show()
3265 rec->flags & FTRACE_FL_REGS ? " R" : " ", in t_show()
3266 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " "); in t_show()
3267 if (rec->flags & FTRACE_FL_TRAMP_EN) { in t_show()
3268 ops = ftrace_find_tramp_ops_any(rec); in t_show()
3276 add_trampoline_func(m, ops, rec); in t_show()
3465 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) in enter_record() argument
3470 entry = ftrace_lookup_ip(hash, rec->ip); in enter_record()
3482 ret = add_hash_entry(hash, rec->ip); in enter_record()
3488 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, in ftrace_match_record() argument
3494 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); in ftrace_match_record()
3531 struct dyn_ftrace *rec; in match_records() local
3557 do_for_each_ftrace_rec(pg, rec) { in match_records()
3558 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) { in match_records()
3559 ret = enter_record(hash, rec, clear_filter); in match_records()
3718 struct dyn_ftrace *rec; in register_ftrace_function_probe() local
3751 do_for_each_ftrace_rec(pg, rec) { in register_ftrace_function_probe()
3753 if (!ftrace_match_record(rec, &func_g, NULL, 0)) in register_ftrace_function_probe()
3774 if (ops->init(ops, rec->ip, &entry->data) < 0) { in register_ftrace_function_probe()
3781 ret = enter_record(hash, rec, 0); in register_ftrace_function_probe()
3789 entry->ip = rec->ip; in register_ftrace_function_probe()
4621 struct dyn_ftrace *rec; in ftrace_set_func() local
4643 do_for_each_ftrace_rec(pg, rec) { in ftrace_set_func()
4645 if (ftrace_match_record(rec, &func_g, NULL, 0)) { in ftrace_set_func()
4649 if (array[i] == rec->ip) { in ftrace_set_func()
4658 array[(*idx)++] = rec->ip; in ftrace_set_func()
4804 struct dyn_ftrace *rec; in ftrace_process_locs() local
4867 rec = &pg->records[pg->index++]; in ftrace_process_locs()
4868 rec->ip = addr; in ftrace_process_locs()
4903 struct dyn_ftrace *rec; in ftrace_release_mod() local
4919 rec = &pg->records[0]; in ftrace_release_mod()
4920 if (within_module_core(rec->ip, mod)) { in ftrace_release_mod()