43 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)  in trace_kprobe_is_return()  argument
45 	return tk->rp.handler != NULL;  in trace_kprobe_is_return()
48 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)  in trace_kprobe_symbol()  argument
50 	return tk->symbol ? tk->symbol : "unknown";  in trace_kprobe_symbol()
53 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)  in trace_kprobe_offset()  argument
55 	return tk->rp.kp.offset;  in trace_kprobe_offset()
58 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)  in trace_kprobe_has_gone()  argument
60 	return !!(kprobe_gone(&tk->rp.kp));  in trace_kprobe_has_gone()
63 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,  in trace_kprobe_within_module()  argument
67 	const char *name = trace_kprobe_symbol(tk);  in trace_kprobe_within_module()
71 static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)  in trace_kprobe_is_on_module()  argument
73 	return !!strchr(trace_kprobe_symbol(tk), ':');  in trace_kprobe_is_on_module()
76 static int register_kprobe_event(struct trace_kprobe *tk);
77 static int unregister_kprobe_event(struct trace_kprobe *tk);
282 	struct trace_kprobe *tk;  in alloc_trace_kprobe()  local
285 	tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);  in alloc_trace_kprobe()
286 	if (!tk)  in alloc_trace_kprobe()
290 		tk->symbol = kstrdup(symbol, GFP_KERNEL);  in alloc_trace_kprobe()
291 		if (!tk->symbol)  in alloc_trace_kprobe()
293 		tk->rp.kp.symbol_name = tk->symbol;  in alloc_trace_kprobe()
294 		tk->rp.kp.offset = offs;  in alloc_trace_kprobe()
296 		tk->rp.kp.addr = addr;  in alloc_trace_kprobe()
299 		tk->rp.handler = kretprobe_dispatcher;  in alloc_trace_kprobe()
301 		tk->rp.kp.pre_handler = kprobe_dispatcher;  in alloc_trace_kprobe()
308 	tk->tp.call.class = &tk->tp.class;  in alloc_trace_kprobe()
309 	tk->tp.call.name = kstrdup(event, GFP_KERNEL);  in alloc_trace_kprobe()
310 	if (!tk->tp.call.name)  in alloc_trace_kprobe()
318 	tk->tp.class.system = kstrdup(group, GFP_KERNEL);  in alloc_trace_kprobe()
319 	if (!tk->tp.class.system)  in alloc_trace_kprobe()
322 	INIT_LIST_HEAD(&tk->list);  in alloc_trace_kprobe()
323 	INIT_LIST_HEAD(&tk->tp.files);  in alloc_trace_kprobe()
324 	return tk;  in alloc_trace_kprobe()
326 	kfree(tk->tp.call.name);  in alloc_trace_kprobe()
327 	kfree(tk->symbol);  in alloc_trace_kprobe()
328 	kfree(tk);  in alloc_trace_kprobe()
332 static void free_trace_kprobe(struct trace_kprobe *tk)  in free_trace_kprobe()  argument
336 	for (i = 0; i < tk->tp.nr_args; i++)  in free_trace_kprobe()
337 		traceprobe_free_probe_arg(&tk->tp.args[i]);  in free_trace_kprobe()
339 	kfree(tk->tp.call.class->system);  in free_trace_kprobe()
340 	kfree(tk->tp.call.name);  in free_trace_kprobe()
341 	kfree(tk->symbol);  in free_trace_kprobe()
342 	kfree(tk);  in free_trace_kprobe()
348 	struct trace_kprobe *tk;  in find_trace_kprobe()  local
350 	list_for_each_entry(tk, &probe_list, list)  in find_trace_kprobe()
351 		if (strcmp(ftrace_event_name(&tk->tp.call), event) == 0 &&  in find_trace_kprobe()
352 		    strcmp(tk->tp.call.class->system, group) == 0)  in find_trace_kprobe()
353 			return tk;  in find_trace_kprobe()
362 enable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)  in enable_trace_kprobe()  argument
376 		list_add_tail_rcu(&link->list, &tk->tp.files);  in enable_trace_kprobe()
378 		tk->tp.flags |= TP_FLAG_TRACE;  in enable_trace_kprobe()
380 		tk->tp.flags |= TP_FLAG_PROFILE;  in enable_trace_kprobe()
382 	if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {  in enable_trace_kprobe()
383 		if (trace_kprobe_is_return(tk))  in enable_trace_kprobe()
384 			ret = enable_kretprobe(&tk->rp);  in enable_trace_kprobe()
386 			ret = enable_kprobe(&tk->rp.kp);  in enable_trace_kprobe()
397 disable_trace_kprobe(struct trace_kprobe *tk, struct ftrace_event_file *file)  in disable_trace_kprobe()  argument
404 		link = find_event_file_link(&tk->tp, file);  in disable_trace_kprobe()
412 		if (!list_empty(&tk->tp.files))  in disable_trace_kprobe()
415 		tk->tp.flags &= ~TP_FLAG_TRACE;  in disable_trace_kprobe()
417 		tk->tp.flags &= ~TP_FLAG_PROFILE;  in disable_trace_kprobe()
419 	if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {  in disable_trace_kprobe()
420 		if (trace_kprobe_is_return(tk))  in disable_trace_kprobe()
421 			disable_kretprobe(&tk->rp);  in disable_trace_kprobe()
423 			disable_kprobe(&tk->rp.kp);  in disable_trace_kprobe()
444 static int __register_trace_kprobe(struct trace_kprobe *tk)  in __register_trace_kprobe()  argument
448 	if (trace_probe_is_registered(&tk->tp))  in __register_trace_kprobe()
451 	for (i = 0; i < tk->tp.nr_args; i++)  in __register_trace_kprobe()
452 		traceprobe_update_arg(&tk->tp.args[i]);  in __register_trace_kprobe()
455 	if (trace_probe_is_enabled(&tk->tp))  in __register_trace_kprobe()
456 		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;  in __register_trace_kprobe()
458 		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;  in __register_trace_kprobe()
460 	if (trace_kprobe_is_return(tk))  in __register_trace_kprobe()
461 		ret = register_kretprobe(&tk->rp);  in __register_trace_kprobe()
463 		ret = register_kprobe(&tk->rp.kp);  in __register_trace_kprobe()
466 		tk->tp.flags |= TP_FLAG_REGISTERED;  in __register_trace_kprobe()
469 			   trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret);  in __register_trace_kprobe()
470 		if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {  in __register_trace_kprobe()
477 				   tk->rp.kp.addr);  in __register_trace_kprobe()
486 static void __unregister_trace_kprobe(struct trace_kprobe *tk)  in __unregister_trace_kprobe()  argument
488 	if (trace_probe_is_registered(&tk->tp)) {  in __unregister_trace_kprobe()
489 		if (trace_kprobe_is_return(tk))  in __unregister_trace_kprobe()
490 			unregister_kretprobe(&tk->rp);  in __unregister_trace_kprobe()
492 			unregister_kprobe(&tk->rp.kp);  in __unregister_trace_kprobe()
493 		tk->tp.flags &= ~TP_FLAG_REGISTERED;  in __unregister_trace_kprobe()
495 		if (tk->rp.kp.symbol_name)  in __unregister_trace_kprobe()
496 			tk->rp.kp.addr = NULL;  in __unregister_trace_kprobe()
501 static int unregister_trace_kprobe(struct trace_kprobe *tk)  in unregister_trace_kprobe()  argument
504 	if (trace_probe_is_enabled(&tk->tp))  in unregister_trace_kprobe()
508 	if (unregister_kprobe_event(tk))  in unregister_trace_kprobe()
511 	__unregister_trace_kprobe(tk);  in unregister_trace_kprobe()
512 	list_del(&tk->list);  in unregister_trace_kprobe()
518 static int register_trace_kprobe(struct trace_kprobe *tk)  in register_trace_kprobe()  argument
526 	old_tk = find_trace_kprobe(ftrace_event_name(&tk->tp.call),  in register_trace_kprobe()
527 			tk->tp.call.class->system);  in register_trace_kprobe()
536 	ret = register_kprobe_event(tk);  in register_trace_kprobe()
543 	ret = __register_trace_kprobe(tk);  in register_trace_kprobe()
545 		unregister_kprobe_event(tk);  in register_trace_kprobe()
547 		list_add_tail(&tk->list, &probe_list);  in register_trace_kprobe()
559 	struct trace_kprobe *tk;  in trace_kprobe_module_callback()  local
567 	list_for_each_entry(tk, &probe_list, list) {  in trace_kprobe_module_callback()
568 		if (trace_kprobe_within_module(tk, mod)) {  in trace_kprobe_module_callback()
570 			__unregister_trace_kprobe(tk);  in trace_kprobe_module_callback()
571 			ret = __register_trace_kprobe(tk);  in trace_kprobe_module_callback()
575 					   ftrace_event_name(&tk->tp.call),  in trace_kprobe_module_callback()
609 	struct trace_kprobe *tk;  in create_trace_kprobe()  local
656 		tk = find_trace_kprobe(event, group);  in create_trace_kprobe()
657 		if (!tk) {  in create_trace_kprobe()
663 		ret = unregister_trace_kprobe(tk);  in create_trace_kprobe()
665 			free_trace_kprobe(tk);  in create_trace_kprobe()
712 	tk = alloc_trace_kprobe(group, event, addr, symbol, offset, argc,  in create_trace_kprobe()
714 	if (IS_ERR(tk)) {  in create_trace_kprobe()
716 			(int)PTR_ERR(tk));  in create_trace_kprobe()
717 		return PTR_ERR(tk);  in create_trace_kprobe()
723 		struct probe_arg *parg = &tk->tp.args[i];  in create_trace_kprobe()
726 		tk->tp.nr_args++;  in create_trace_kprobe()
754 							tk->tp.args, i)) {  in create_trace_kprobe()
762 		ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,  in create_trace_kprobe()
771 	ret = register_trace_kprobe(tk);  in create_trace_kprobe()
777 	free_trace_kprobe(tk);  in create_trace_kprobe()
783 	struct trace_kprobe *tk;  in release_all_trace_kprobes()  local
788 	list_for_each_entry(tk, &probe_list, list)  in release_all_trace_kprobes()
789 		if (trace_probe_is_enabled(&tk->tp)) {  in release_all_trace_kprobes()
795 		tk = list_entry(probe_list.next, struct trace_kprobe, list);  in release_all_trace_kprobes()
796 		ret = unregister_trace_kprobe(tk);  in release_all_trace_kprobes()
799 		free_trace_kprobe(tk);  in release_all_trace_kprobes()
827 	struct trace_kprobe *tk = v;  in probes_seq_show()  local
830 	seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');  in probes_seq_show()
831 	seq_printf(m, ":%s/%s", tk->tp.call.class->system,  in probes_seq_show()
832 			ftrace_event_name(&tk->tp.call));  in probes_seq_show()
834 	if (!tk->symbol)  in probes_seq_show()
835 		seq_printf(m, " 0x%p", tk->rp.kp.addr);  in probes_seq_show()
836 	else if (tk->rp.kp.offset)  in probes_seq_show()
837 		seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),  in probes_seq_show()
838 			   tk->rp.kp.offset);  in probes_seq_show()
840 		seq_printf(m, " %s", trace_kprobe_symbol(tk));  in probes_seq_show()
842 	for (i = 0; i < tk->tp.nr_args; i++)  in probes_seq_show()
843 		seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);  in probes_seq_show()
888 	struct trace_kprobe *tk = v;  in probes_profile_seq_show()  local
891 		   ftrace_event_name(&tk->tp.call), tk->nhit,  in probes_profile_seq_show()
892 		   tk->rp.kp.nmissed);  in probes_profile_seq_show()
919 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,  in __kprobe_trace_func()  argument
927 	struct ftrace_event_call *call = &tk->tp.call;  in __kprobe_trace_func()
937 	dsize = __get_data_size(&tk->tp, regs);  in __kprobe_trace_func()
938 	size = sizeof(*entry) + tk->tp.size + dsize;  in __kprobe_trace_func()
947 	entry->ip = (unsigned long)tk->rp.kp.addr;  in __kprobe_trace_func()
948 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in __kprobe_trace_func()
955 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)  in kprobe_trace_func()  argument
959 	list_for_each_entry_rcu(link, &tk->tp.files, list)  in kprobe_trace_func()
960 		__kprobe_trace_func(tk, regs, link->file);  in kprobe_trace_func()
966 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,  in __kretprobe_trace_func()  argument
975 	struct ftrace_event_call *call = &tk->tp.call;  in __kretprobe_trace_func()
985 	dsize = __get_data_size(&tk->tp, regs);  in __kretprobe_trace_func()
986 	size = sizeof(*entry) + tk->tp.size + dsize;  in __kretprobe_trace_func()
995 	entry->func = (unsigned long)tk->rp.kp.addr;  in __kretprobe_trace_func()
997 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in __kretprobe_trace_func()
1004 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,  in kretprobe_trace_func()  argument
1009 	list_for_each_entry_rcu(link, &tk->tp.files, list)  in kretprobe_trace_func()
1010 		__kretprobe_trace_func(tk, ri, regs, link->file);  in kretprobe_trace_func()
1088 	struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;  in kprobe_event_define_fields()  local
1092 	for (i = 0; i < tk->tp.nr_args; i++) {  in kprobe_event_define_fields()
1093 		struct probe_arg *parg = &tk->tp.args[i];  in kprobe_event_define_fields()
1111 	struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;  in kretprobe_event_define_fields()  local
1116 	for (i = 0; i < tk->tp.nr_args; i++) {  in kretprobe_event_define_fields()
1117 		struct probe_arg *parg = &tk->tp.args[i];  in kretprobe_event_define_fields()
1135 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)  in kprobe_perf_func()  argument
1137 	struct ftrace_event_call *call = &tk->tp.call;  in kprobe_perf_func()
1151 	dsize = __get_data_size(&tk->tp, regs);  in kprobe_perf_func()
1152 	__size = sizeof(*entry) + tk->tp.size + dsize;  in kprobe_perf_func()
1160 	entry->ip = (unsigned long)tk->rp.kp.addr;  in kprobe_perf_func()
1162 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in kprobe_perf_func()
1169 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,  in kretprobe_perf_func()  argument
1172 	struct ftrace_event_call *call = &tk->tp.call;  in kretprobe_perf_func()
1186 	dsize = __get_data_size(&tk->tp, regs);  in kretprobe_perf_func()
1187 	__size = sizeof(*entry) + tk->tp.size + dsize;  in kretprobe_perf_func()
1195 	entry->func = (unsigned long)tk->rp.kp.addr;  in kretprobe_perf_func()
1197 	store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);  in kretprobe_perf_func()
1212 	struct trace_kprobe *tk = (struct trace_kprobe *)event->data;  in kprobe_register()  local
1217 		return enable_trace_kprobe(tk, file);  in kprobe_register()
1219 		return disable_trace_kprobe(tk, file);  in kprobe_register()
1223 		return enable_trace_kprobe(tk, NULL);  in kprobe_register()
1225 		return disable_trace_kprobe(tk, NULL);  in kprobe_register()
1238 	struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);  in kprobe_dispatcher()  local
1240 	tk->nhit++;  in kprobe_dispatcher()
1242 	if (tk->tp.flags & TP_FLAG_TRACE)  in kprobe_dispatcher()
1243 		kprobe_trace_func(tk, regs);  in kprobe_dispatcher()
1245 	if (tk->tp.flags & TP_FLAG_PROFILE)  in kprobe_dispatcher()
1246 		kprobe_perf_func(tk, regs);  in kprobe_dispatcher()
1255 	struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);  in kretprobe_dispatcher()  local
1257 	tk->nhit++;  in kretprobe_dispatcher()
1259 	if (tk->tp.flags & TP_FLAG_TRACE)  in kretprobe_dispatcher()
1260 		kretprobe_trace_func(tk, ri, regs);  in kretprobe_dispatcher()
1262 	if (tk->tp.flags & TP_FLAG_PROFILE)  in kretprobe_dispatcher()
1263 		kretprobe_perf_func(tk, ri, regs);  in kretprobe_dispatcher()
1277 static int register_kprobe_event(struct trace_kprobe *tk)  in register_kprobe_event()  argument
1279 	struct ftrace_event_call *call = &tk->tp.call;  in register_kprobe_event()
1284 	if (trace_kprobe_is_return(tk)) {  in register_kprobe_event()
1291 	if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)  in register_kprobe_event()
1300 	call->data = tk;  in register_kprobe_event()
1311 static int unregister_kprobe_event(struct trace_kprobe *tk)  in unregister_kprobe_event()  argument
1316 	ret = trace_remove_event_call(&tk->tp.call);  in unregister_kprobe_event()
1318 		kfree(tk->tp.call.print_fmt);  in unregister_kprobe_event()
1368 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)  in find_trace_probe_file()  argument
1373 		if (file->event_call == &tk->tp.call)  in find_trace_probe_file()
1387 	struct trace_kprobe *tk;  in kprobe_trace_self_tests_init()  local
1405 		tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1406 		if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1410 			file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1415 				enable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()
1426 		tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1427 		if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1431 			file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1436 				enable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()
1446 	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1447 	if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1451 		file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1456 			disable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()
1459 	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);  in kprobe_trace_self_tests_init()
1460 	if (WARN_ON_ONCE(tk == NULL)) {  in kprobe_trace_self_tests_init()
1464 		file = find_trace_probe_file(tk, top_trace_array());  in kprobe_trace_self_tests_init()
1469 			disable_trace_kprobe(tk, file);  in kprobe_trace_self_tests_init()