140 static inline struct lock_class *hlock_class(struct held_lock *hlock)  in hlock_class()  argument
142 	if (!hlock->class_idx) {  in hlock_class()
149 	return lock_classes + hlock->class_idx - 1;  in hlock_class()
257 static void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
265 	holdtime = lockstat_clock() - hlock->holdtime_stamp;  in lock_release_holdtime()
267 	stats = get_lock_stats(hlock_class(hlock));  in lock_release_holdtime()
268 	if (hlock->read)  in lock_release_holdtime()
275 static inline void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
552 static void print_lock(struct held_lock *hlock)  in print_lock()  argument
558 	unsigned int class_idx = hlock->class_idx;  in print_lock()
570 	print_ip_sym(hlock->acquire_ip);  in print_lock()
1933 	struct held_lock *hlock;  in check_prevs_add()  local
1952 		hlock = curr->held_locks + depth - 1;  in check_prevs_add()
1957 		if (hlock->read != 2 && hlock->check) {  in check_prevs_add()
1958 			if (!check_prev_add(curr, hlock, next,  in check_prevs_add()
1967 			if (!hlock->trylock)  in check_prevs_add()
2016 				     struct held_lock *hlock,  in lookup_chain_cache()  argument
2019 	struct lock_class *class = hlock_class(hlock);  in lookup_chain_cache()
2076 	chain->irq_context = hlock->irq_context;  in lookup_chain_cache()
2080 		if (hlock_curr->irq_context != hlock->irq_context)  in lookup_chain_cache()
2102 		struct held_lock *hlock, int chain_head, u64 chain_key)  in validate_chain()  argument
2114 	if (!hlock->trylock && hlock->check &&  in validate_chain()
2115 	    lookup_chain_cache(curr, hlock, chain_key)) {  in validate_chain()
2128 		int ret = check_deadlock(curr, hlock, lock, hlock->read);  in validate_chain()
2138 			hlock->read = 2;  in validate_chain()
2144 			if (!check_prevs_add(curr, hlock))  in validate_chain()
2156 	       	struct lockdep_map *lock, struct held_lock *hlock,  in validate_chain()  argument
2170 	struct held_lock *hlock, *prev_hlock = NULL;  in check_chain_key()  local
2175 		hlock = curr->held_locks + i;  in check_chain_key()
2176 		if (chain_key != hlock->prev_chain_key) {  in check_chain_key()
2185 				(unsigned long long)hlock->prev_chain_key);  in check_chain_key()
2188 		id = hlock->class_idx - 1;  in check_chain_key()
2196 							hlock->irq_context))  in check_chain_key()
2199 		prev_hlock = hlock;  in check_chain_key()
2521 	struct held_lock *hlock;  in mark_held_locks()  local
2525 		hlock = curr->held_locks + i;  in mark_held_locks()
2528 		if (hlock->read)  in mark_held_locks()
2533 		if (!hlock->check)  in mark_held_locks()
2536 		if (!mark_lock(curr, hlock, usage_bit))  in mark_held_locks()
2778 static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)  in mark_irqflags()  argument
2784 	if (!hlock->trylock) {  in mark_irqflags()
2785 		if (hlock->read) {  in mark_irqflags()
2787 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2791 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2796 				if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ))  in mark_irqflags()
2799 				if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ))  in mark_irqflags()
2803 	if (!hlock->hardirqs_off) {  in mark_irqflags()
2804 		if (hlock->read) {  in mark_irqflags()
2805 			if (!mark_lock(curr, hlock,  in mark_irqflags()
2809 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2813 			if (!mark_lock(curr, hlock,  in mark_irqflags()
2817 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2829 	if (!hlock->trylock && (curr->lockdep_reclaim_gfp & __GFP_FS)) {  in mark_irqflags()
2830 		if (hlock->read) {  in mark_irqflags()
2831 			if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS_READ))  in mark_irqflags()
2834 			if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS))  in mark_irqflags()
2843 		struct held_lock *hlock)  in separate_irq_context()  argument
2850 	hlock->irq_context = 2*(curr->hardirq_context ? 1 : 0) +  in separate_irq_context()
2861 		if (prev_hlock->irq_context != hlock->irq_context)  in separate_irq_context()
2878 		struct held_lock *hlock)  in mark_irqflags()  argument
2884 		struct held_lock *hlock)  in separate_irq_context()  argument
3030 				struct held_lock *hlock,  in print_lock_nested_lock_not_held()  argument
3045 	print_lock(hlock);  in print_lock_nested_lock_not_held()
3048 	printk("%s\n", hlock->nest_lock->name);  in print_lock_nested_lock_not_held()
3075 	struct held_lock *hlock;  in __lock_acquire()  local
3129 		hlock = curr->held_locks + depth - 1;  in __lock_acquire()
3130 		if (hlock->class_idx == class_idx && nest_lock) {  in __lock_acquire()
3131 			if (hlock->references)  in __lock_acquire()
3132 				hlock->references++;  in __lock_acquire()
3134 				hlock->references = 2;  in __lock_acquire()
3140 	hlock = curr->held_locks + depth;  in __lock_acquire()
3147 	hlock->class_idx = class_idx;  in __lock_acquire()
3148 	hlock->acquire_ip = ip;  in __lock_acquire()
3149 	hlock->instance = lock;  in __lock_acquire()
3150 	hlock->nest_lock = nest_lock;  in __lock_acquire()
3151 	hlock->trylock = trylock;  in __lock_acquire()
3152 	hlock->read = read;  in __lock_acquire()
3153 	hlock->check = check;  in __lock_acquire()
3154 	hlock->hardirqs_off = !!hardirqs_off;  in __lock_acquire()
3155 	hlock->references = references;  in __lock_acquire()
3157 	hlock->waittime_stamp = 0;  in __lock_acquire()
3158 	hlock->holdtime_stamp = lockstat_clock();  in __lock_acquire()
3160 	hlock->pin_count = pin_count;  in __lock_acquire()
3162 	if (check && !mark_irqflags(curr, hlock))  in __lock_acquire()
3166 	if (!mark_lock(curr, hlock, LOCK_USED))  in __lock_acquire()
3196 	hlock->prev_chain_key = chain_key;  in __lock_acquire()
3197 	if (separate_irq_context(curr, hlock)) {  in __lock_acquire()
3204 		return print_lock_nested_lock_not_held(curr, hlock, ip);  in __lock_acquire()
3206 	if (!validate_chain(curr, lock, hlock, chain_head, chain_key))  in __lock_acquire()
3264 static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)  in match_held_lock()  argument
3266 	if (hlock->instance == lock)  in match_held_lock()
3269 	if (hlock->references) {  in match_held_lock()
3289 		if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))  in match_held_lock()
3292 		if (hlock->class_idx == class - lock_classes + 1)  in match_held_lock()
3305 	struct held_lock *hlock, *prev_hlock;  in __lock_set_class()  local
3320 		hlock = curr->held_locks + i;  in __lock_set_class()
3324 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_set_class()
3326 		if (match_held_lock(hlock, lock))  in __lock_set_class()
3328 		prev_hlock = hlock;  in __lock_set_class()
3335 	hlock->class_idx = class - lock_classes + 1;  in __lock_set_class()
3338 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_set_class()
3341 		hlock = curr->held_locks + i;  in __lock_set_class()
3342 		if (!__lock_acquire(hlock->instance,  in __lock_set_class()
3343 			hlock_class(hlock)->subclass, hlock->trylock,  in __lock_set_class()
3344 				hlock->read, hlock->check, hlock->hardirqs_off,  in __lock_set_class()
3345 				hlock->nest_lock, hlock->acquire_ip,  in __lock_set_class()
3346 				hlock->references, hlock->pin_count))  in __lock_set_class()
3370 	struct held_lock *hlock, *prev_hlock;  in __lock_release()  local
3391 		hlock = curr->held_locks + i;  in __lock_release()
3395 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_release()
3397 		if (match_held_lock(hlock, lock))  in __lock_release()
3399 		prev_hlock = hlock;  in __lock_release()
3404 	if (hlock->instance == lock)  in __lock_release()
3405 		lock_release_holdtime(hlock);  in __lock_release()
3407 	WARN(hlock->pin_count, "releasing a pinned lock\n");  in __lock_release()
3409 	if (hlock->references) {  in __lock_release()
3410 		hlock->references--;  in __lock_release()
3411 		if (hlock->references) {  in __lock_release()
3428 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_release()
3431 		hlock = curr->held_locks + i;  in __lock_release()
3432 		if (!__lock_acquire(hlock->instance,  in __lock_release()
3433 			hlock_class(hlock)->subclass, hlock->trylock,  in __lock_release()
3434 				hlock->read, hlock->check, hlock->hardirqs_off,  in __lock_release()
3435 				hlock->nest_lock, hlock->acquire_ip,  in __lock_release()
3436 				hlock->references, hlock->pin_count))  in __lock_release()
3456 		struct held_lock *hlock = curr->held_locks + i;  in __lock_is_held()  local
3458 		if (match_held_lock(hlock, lock))  in __lock_is_held()
3474 		struct held_lock *hlock = curr->held_locks + i;  in __lock_pin_lock()  local
3476 		if (match_held_lock(hlock, lock)) {  in __lock_pin_lock()
3477 			hlock->pin_count++;  in __lock_pin_lock()
3494 		struct held_lock *hlock = curr->held_locks + i;  in __lock_unpin_lock()  local
3496 		if (match_held_lock(hlock, lock)) {  in __lock_unpin_lock()
3497 			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))  in __lock_unpin_lock()
3500 			hlock->pin_count--;  in __lock_unpin_lock()
3709 	struct held_lock *hlock, *prev_hlock;  in __lock_contended()  local
3724 		hlock = curr->held_locks + i;  in __lock_contended()
3728 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_contended()
3730 		if (match_held_lock(hlock, lock))  in __lock_contended()
3732 		prev_hlock = hlock;  in __lock_contended()
3738 	if (hlock->instance != lock)  in __lock_contended()
3741 	hlock->waittime_stamp = lockstat_clock();  in __lock_contended()
3743 	contention_point = lock_point(hlock_class(hlock)->contention_point, ip);  in __lock_contended()
3744 	contending_point = lock_point(hlock_class(hlock)->contending_point,  in __lock_contended()
3747 	stats = get_lock_stats(hlock_class(hlock));  in __lock_contended()
3753 		stats->bounces[bounce_contended + !!hlock->read]++;  in __lock_contended()
3761 	struct held_lock *hlock, *prev_hlock;  in __lock_acquired()  local
3777 		hlock = curr->held_locks + i;  in __lock_acquired()
3781 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_acquired()
3783 		if (match_held_lock(hlock, lock))  in __lock_acquired()
3785 		prev_hlock = hlock;  in __lock_acquired()
3791 	if (hlock->instance != lock)  in __lock_acquired()
3795 	if (hlock->waittime_stamp) {  in __lock_acquired()
3797 		waittime = now - hlock->waittime_stamp;  in __lock_acquired()
3798 		hlock->holdtime_stamp = now;  in __lock_acquired()
3803 	stats = get_lock_stats(hlock_class(hlock));  in __lock_acquired()
3805 		if (hlock->read)  in __lock_acquired()
3811 		stats->bounces[bounce_acquired + !!hlock->read]++;  in __lock_acquired()
4076 		     const void *mem_to, struct held_lock *hlock)  in print_freed_lock_bug()  argument
4090 	print_lock(hlock);  in print_freed_lock_bug()
4112 	struct held_lock *hlock;  in debug_check_no_locks_freed()  local
4121 		hlock = curr->held_locks + i;  in debug_check_no_locks_freed()
4123 		if (not_in_range(mem_from, mem_len, hlock->instance,  in debug_check_no_locks_freed()
4124 					sizeof(*hlock->instance)))  in debug_check_no_locks_freed()
4127 		print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);  in debug_check_no_locks_freed()