514 	struct hlist_head		*hash;  member
686 	memset(stat->hash, 0,  in ftrace_profile_reset()
753 	if (stat->hash) {  in ftrace_profile_init_cpu()
765 	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);  in ftrace_profile_init_cpu()
767 	if (!stat->hash)  in ftrace_profile_init_cpu()
772 		kfree(stat->hash);  in ftrace_profile_init_cpu()
773 		stat->hash = NULL;  in ftrace_profile_init_cpu()
803 	hhd = &stat->hash[key];  in ftrace_find_profiled_func()
822 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);  in ftrace_add_profile()
875 	if (!stat->hash || !ftrace_profile_enabled)  in function_profile_call()
906 	if (!stat->hash || !ftrace_profile_enabled)  in profile_graph_return()
1210 static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash)  in ftrace_hash_empty()  argument
1212 	return !hash || !hash->count;  in ftrace_hash_empty()
1216 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)  in ftrace_lookup_ip()  argument
1222 	if (ftrace_hash_empty(hash))  in ftrace_lookup_ip()
1225 	if (hash->size_bits > 0)  in ftrace_lookup_ip()
1226 		key = hash_long(ip, hash->size_bits);  in ftrace_lookup_ip()
1230 	hhd = &hash->buckets[key];  in ftrace_lookup_ip()
1239 static void __add_hash_entry(struct ftrace_hash *hash,  in __add_hash_entry()  argument
1245 	if (hash->size_bits)  in __add_hash_entry()
1246 		key = hash_long(entry->ip, hash->size_bits);  in __add_hash_entry()
1250 	hhd = &hash->buckets[key];  in __add_hash_entry()
1252 	hash->count++;  in __add_hash_entry()
1255 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)  in add_hash_entry()  argument
1264 	__add_hash_entry(hash, entry);  in add_hash_entry()
1270 free_hash_entry(struct ftrace_hash *hash,  in free_hash_entry()  argument
1275 	hash->count--;  in free_hash_entry()
1279 remove_hash_entry(struct ftrace_hash *hash,  in remove_hash_entry()  argument
1283 	hash->count--;  in remove_hash_entry()
1286 static void ftrace_hash_clear(struct ftrace_hash *hash)  in ftrace_hash_clear()  argument
1291 	int size = 1 << hash->size_bits;  in ftrace_hash_clear()
1294 	if (!hash->count)  in ftrace_hash_clear()
1298 		hhd = &hash->buckets[i];  in ftrace_hash_clear()
1300 			free_hash_entry(hash, entry);  in ftrace_hash_clear()
1302 	FTRACE_WARN_ON(hash->count);  in ftrace_hash_clear()
1305 static void free_ftrace_hash(struct ftrace_hash *hash)  in free_ftrace_hash()  argument
1307 	if (!hash || hash == EMPTY_HASH)  in free_ftrace_hash()
1309 	ftrace_hash_clear(hash);  in free_ftrace_hash()
1310 	kfree(hash->buckets);  in free_ftrace_hash()
1311 	kfree(hash);  in free_ftrace_hash()
1316 	struct ftrace_hash *hash;  in __free_ftrace_hash_rcu()  local
1318 	hash = container_of(rcu, struct ftrace_hash, rcu);  in __free_ftrace_hash_rcu()
1319 	free_ftrace_hash(hash);  in __free_ftrace_hash_rcu()
1322 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)  in free_ftrace_hash_rcu()  argument
1324 	if (!hash || hash == EMPTY_HASH)  in free_ftrace_hash_rcu()
1326 	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);  in free_ftrace_hash_rcu()
1338 	struct ftrace_hash *hash;  in alloc_ftrace_hash()  local
1341 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);  in alloc_ftrace_hash()
1342 	if (!hash)  in alloc_ftrace_hash()
1346 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);  in alloc_ftrace_hash()
1348 	if (!hash->buckets) {  in alloc_ftrace_hash()
1349 		kfree(hash);  in alloc_ftrace_hash()
1353 	hash->size_bits = size_bits;  in alloc_ftrace_hash()
1355 	return hash;  in alloc_ftrace_hash()
1359 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)  in alloc_and_copy_ftrace_hash()  argument
1372 	if (ftrace_hash_empty(hash))  in alloc_and_copy_ftrace_hash()
1375 	size = 1 << hash->size_bits;  in alloc_and_copy_ftrace_hash()
1377 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {  in alloc_and_copy_ftrace_hash()
1384 	FTRACE_WARN_ON(new_hash->count != hash->count);  in alloc_and_copy_ftrace_hash()
1475 			     struct ftrace_ops_hash *hash)  in hash_contains_ip()  argument
1483 	return (ftrace_hash_empty(hash->filter_hash) ||  in hash_contains_ip()
1484 		ftrace_lookup_ip(hash->filter_hash, ip)) &&  in hash_contains_ip()
1485 		(ftrace_hash_empty(hash->notrace_hash) ||  in hash_contains_ip()
1486 		 !ftrace_lookup_ip(hash->notrace_hash, ip));  in hash_contains_ip()
1504 	struct ftrace_ops_hash hash;  in ftrace_ops_test()  local
1517 	hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);  in ftrace_ops_test()
1518 	hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);  in ftrace_ops_test()
1520 	if (hash_contains_ip(ip, &hash))  in ftrace_ops_test()
1636 	struct ftrace_hash *hash;  in __ftrace_hash_rec_update()  local
1659 		hash = ops->func_hash->filter_hash;  in __ftrace_hash_rec_update()
1661 		if (ftrace_hash_empty(hash))  in __ftrace_hash_rec_update()
1665 		hash = ops->func_hash->notrace_hash;  in __ftrace_hash_rec_update()
1671 		if (ftrace_hash_empty(hash))  in __ftrace_hash_rec_update()
1688 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);  in __ftrace_hash_rec_update()
1773 		if (!all && count == hash->count)  in __ftrace_hash_rec_update()
1900 	struct ftrace_hash *hash = ops->func_hash->filter_hash;  in ftrace_hash_ipmodify_enable()  local
1902 	if (ftrace_hash_empty(hash))  in ftrace_hash_ipmodify_enable()
1903 		hash = NULL;  in ftrace_hash_ipmodify_enable()
1905 	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);  in ftrace_hash_ipmodify_enable()
1911 	struct ftrace_hash *hash = ops->func_hash->filter_hash;  in ftrace_hash_ipmodify_disable()  local
1913 	if (ftrace_hash_empty(hash))  in ftrace_hash_ipmodify_disable()
1914 		hash = NULL;  in ftrace_hash_ipmodify_disable()
1916 	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);  in ftrace_hash_ipmodify_disable()
2996 	struct ftrace_hash		*hash;  member
3334 	struct ftrace_hash *hash;  in ftrace_regex_open()  local
3357 		hash = ops->func_hash->notrace_hash;  in ftrace_regex_open()
3359 		hash = ops->func_hash->filter_hash;  in ftrace_regex_open()
3365 			iter->hash = alloc_ftrace_hash(size_bits);  in ftrace_regex_open()
3367 			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);  in ftrace_regex_open()
3369 		if (!iter->hash) {  in ftrace_regex_open()
3386 			free_ftrace_hash(iter->hash);  in ftrace_regex_open()
3447 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)  in enter_record()  argument
3452 	entry = ftrace_lookup_ip(hash, rec->ip);  in enter_record()
3458 		free_hash_entry(hash, entry);  in enter_record()
3464 		ret = add_hash_entry(hash, rec->ip);  in enter_record()
3492 match_records(struct ftrace_hash *hash, char *buff,  in match_records()  argument
3515 			ret = enter_record(hash, rec, not);  in match_records()
3530 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)  in ftrace_match_records()  argument
3532 	return match_records(hash, buff, len, NULL, 0);  in ftrace_match_records()
3536 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)  in ftrace_match_module_records()  argument
3550 	return match_records(hash, buff, strlen(buff), mod, not);  in ftrace_match_module_records()
3559 ftrace_mod_callback(struct ftrace_hash *hash,  in ftrace_mod_callback()  argument
3581 	ret = ftrace_match_module_records(hash, func, mod);  in ftrace_mod_callback()
3699 	struct ftrace_hash *hash;  in register_ftrace_function_probe()  local
3721 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);  in register_ftrace_function_probe()
3722 	if (!hash) {  in register_ftrace_function_probe()
3764 		ret = enter_record(hash, rec, 0);  in register_ftrace_function_probe()
3779 	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);  in register_ftrace_function_probe()
3792 	free_ftrace_hash(hash);  in register_ftrace_function_probe()
3812 	struct ftrace_hash *hash;  in __unregister_ftrace_function_probe()  local
3835 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);  in __unregister_ftrace_function_probe()
3836 	if (!hash)  in __unregister_ftrace_function_probe()
3862 			rec_entry = ftrace_lookup_ip(hash, entry->ip);  in __unregister_ftrace_function_probe()
3865 				free_hash_entry(hash, rec_entry);  in __unregister_ftrace_function_probe()
3877 	ret = ftrace_hash_move(&trace_probe_ops, 1, orig_hash, hash);  in __unregister_ftrace_function_probe()
3890 	free_ftrace_hash(hash);  in __unregister_ftrace_function_probe()
3961 static int ftrace_process_regex(struct ftrace_hash *hash,  in ftrace_process_regex()  argument
3971 		ret = ftrace_match_records(hash, func, len);  in ftrace_process_regex()
3986 			ret = p->func(hash, func, command, next, enable);  in ftrace_process_regex()
4023 		ret = ftrace_process_regex(iter->hash, parser->buffer,  in ftrace_regex_write()
4050 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)  in ftrace_match_addr()  argument
4058 		entry = ftrace_lookup_ip(hash, ip);  in ftrace_match_addr()
4061 		free_hash_entry(hash, entry);  in ftrace_match_addr()
4065 	return add_hash_entry(hash, ip);  in ftrace_match_addr()
4106 	struct ftrace_hash *hash;  in ftrace_set_hash()  local
4120 		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);  in ftrace_set_hash()
4122 		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);  in ftrace_set_hash()
4124 	if (!hash) {  in ftrace_set_hash()
4129 	if (buf && !ftrace_match_records(hash, buf, len)) {  in ftrace_set_hash()
4134 		ret = ftrace_match_addr(hash, ip, remove);  in ftrace_set_hash()
4143 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);  in ftrace_set_hash()
4153 	free_ftrace_hash(hash);  in ftrace_set_hash()
4374 		ftrace_match_records(iter->hash, parser->buffer, parser->idx);  in ftrace_regex_release()
4394 				       orig_hash, iter->hash);  in ftrace_regex_release()
4403 	free_ftrace_hash(iter->hash);  in ftrace_regex_release()