140 static inline struct lock_class *hlock_class(struct held_lock *hlock)  in hlock_class()  argument
142 	if (!hlock->class_idx) {  in hlock_class()
149 	return lock_classes + hlock->class_idx - 1;  in hlock_class()
257 static void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
265 	holdtime = lockstat_clock() - hlock->holdtime_stamp;  in lock_release_holdtime()
267 	stats = get_lock_stats(hlock_class(hlock));  in lock_release_holdtime()
268 	if (hlock->read)  in lock_release_holdtime()
275 static inline void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
552 static void print_lock(struct held_lock *hlock)  in print_lock()  argument
558 	unsigned int class_idx = hlock->class_idx;  in print_lock()
570 	print_ip_sym(hlock->acquire_ip);  in print_lock()
1933 	struct held_lock *hlock;  in check_prevs_add()  local
1952 		hlock = curr->held_locks + depth - 1;  in check_prevs_add()
1957 		if (hlock->read != 2 && hlock->check) {  in check_prevs_add()
1958 			if (!check_prev_add(curr, hlock, next,  in check_prevs_add()
1967 			if (!hlock->trylock)  in check_prevs_add()
2016 				     struct held_lock *hlock,  in lookup_chain_cache()  argument
2019 	struct lock_class *class = hlock_class(hlock);  in lookup_chain_cache()
2076 	chain->irq_context = hlock->irq_context;  in lookup_chain_cache()
2080 		if (hlock_curr->irq_context != hlock->irq_context)  in lookup_chain_cache()
2102 		struct held_lock *hlock, int chain_head, u64 chain_key)  in validate_chain()  argument
2114 	if (!hlock->trylock && hlock->check &&  in validate_chain()
2115 	    lookup_chain_cache(curr, hlock, chain_key)) {  in validate_chain()
2128 		int ret = check_deadlock(curr, hlock, lock, hlock->read);  in validate_chain()
2138 			hlock->read = 2;  in validate_chain()
2144 			if (!check_prevs_add(curr, hlock))  in validate_chain()
2156 	       	struct lockdep_map *lock, struct held_lock *hlock,  in validate_chain()  argument
2170 	struct held_lock *hlock, *prev_hlock = NULL;  in check_chain_key()  local
2175 		hlock = curr->held_locks + i;  in check_chain_key()
2176 		if (chain_key != hlock->prev_chain_key) {  in check_chain_key()
2185 				(unsigned long long)hlock->prev_chain_key);  in check_chain_key()
2188 		id = hlock->class_idx - 1;  in check_chain_key()
2196 							hlock->irq_context))  in check_chain_key()
2199 		prev_hlock = hlock;  in check_chain_key()
2521 	struct held_lock *hlock;  in mark_held_locks()  local
2525 		hlock = curr->held_locks + i;  in mark_held_locks()
2528 		if (hlock->read)  in mark_held_locks()
2533 		if (!hlock->check)  in mark_held_locks()
2536 		if (!mark_lock(curr, hlock, usage_bit))  in mark_held_locks()
2778 static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)  in mark_irqflags()  argument
2784 	if (!hlock->trylock) {  in mark_irqflags()
2785 		if (hlock->read) {  in mark_irqflags()
2787 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2791 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2796 				if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ))  in mark_irqflags()
2799 				if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ))  in mark_irqflags()
2803 	if (!hlock->hardirqs_off) {  in mark_irqflags()
2804 		if (hlock->read) {  in mark_irqflags()
2805 			if (!mark_lock(curr, hlock,  in mark_irqflags()
2809 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2813 			if (!mark_lock(curr, hlock,  in mark_irqflags()
2817 				if (!mark_lock(curr, hlock,  in mark_irqflags()
2829 	if (!hlock->trylock && (curr->lockdep_reclaim_gfp & __GFP_FS)) {  in mark_irqflags()
2830 		if (hlock->read) {  in mark_irqflags()
2831 			if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS_READ))  in mark_irqflags()
2834 			if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS))  in mark_irqflags()
2843 		struct held_lock *hlock)  in separate_irq_context()  argument
2850 	hlock->irq_context = 2*(curr->hardirq_context ? 1 : 0) +  in separate_irq_context()
2861 		if (prev_hlock->irq_context != hlock->irq_context)  in separate_irq_context()
2878 		struct held_lock *hlock)  in mark_irqflags()  argument
2884 		struct held_lock *hlock)  in separate_irq_context()  argument
3030 				struct held_lock *hlock,  in print_lock_nested_lock_not_held()  argument
3045 	print_lock(hlock);  in print_lock_nested_lock_not_held()
3048 	printk("%s\n", hlock->nest_lock->name);  in print_lock_nested_lock_not_held()
3075 	struct held_lock *hlock;  in __lock_acquire()  local
3129 		hlock = curr->held_locks + depth - 1;  in __lock_acquire()
3130 		if (hlock->class_idx == class_idx && nest_lock) {  in __lock_acquire()
3131 			if (hlock->references)  in __lock_acquire()
3132 				hlock->references++;  in __lock_acquire()
3134 				hlock->references = 2;  in __lock_acquire()
3140 	hlock = curr->held_locks + depth;  in __lock_acquire()
3147 	hlock->class_idx = class_idx;  in __lock_acquire()
3148 	hlock->acquire_ip = ip;  in __lock_acquire()
3149 	hlock->instance = lock;  in __lock_acquire()
3150 	hlock->nest_lock = nest_lock;  in __lock_acquire()
3151 	hlock->trylock = trylock;  in __lock_acquire()
3152 	hlock->read = read;  in __lock_acquire()
3153 	hlock->check = check;  in __lock_acquire()
3154 	hlock->hardirqs_off = !!hardirqs_off;  in __lock_acquire()
3155 	hlock->references = references;  in __lock_acquire()
3157 	hlock->waittime_stamp = 0;  in __lock_acquire()
3158 	hlock->holdtime_stamp = lockstat_clock();  in __lock_acquire()
3161 	if (check && !mark_irqflags(curr, hlock))  in __lock_acquire()
3165 	if (!mark_lock(curr, hlock, LOCK_USED))  in __lock_acquire()
3195 	hlock->prev_chain_key = chain_key;  in __lock_acquire()
3196 	if (separate_irq_context(curr, hlock)) {  in __lock_acquire()
3203 		return print_lock_nested_lock_not_held(curr, hlock, ip);  in __lock_acquire()
3205 	if (!validate_chain(curr, lock, hlock, chain_head, chain_key))  in __lock_acquire()
3283 static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)  in match_held_lock()  argument
3285 	if (hlock->instance == lock)  in match_held_lock()
3288 	if (hlock->references) {  in match_held_lock()
3308 		if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))  in match_held_lock()
3311 		if (hlock->class_idx == class - lock_classes + 1)  in match_held_lock()
3324 	struct held_lock *hlock, *prev_hlock;  in __lock_set_class()  local
3339 		hlock = curr->held_locks + i;  in __lock_set_class()
3343 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_set_class()
3345 		if (match_held_lock(hlock, lock))  in __lock_set_class()
3347 		prev_hlock = hlock;  in __lock_set_class()
3354 	hlock->class_idx = class - lock_classes + 1;  in __lock_set_class()
3357 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_set_class()
3360 		hlock = curr->held_locks + i;  in __lock_set_class()
3361 		if (!__lock_acquire(hlock->instance,  in __lock_set_class()
3362 			hlock_class(hlock)->subclass, hlock->trylock,  in __lock_set_class()
3363 				hlock->read, hlock->check, hlock->hardirqs_off,  in __lock_set_class()
3364 				hlock->nest_lock, hlock->acquire_ip,  in __lock_set_class()
3365 				hlock->references))  in __lock_set_class()
3388 	struct held_lock *hlock, *prev_hlock;  in lock_release_non_nested()  local
3406 		hlock = curr->held_locks + i;  in lock_release_non_nested()
3410 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in lock_release_non_nested()
3412 		if (match_held_lock(hlock, lock))  in lock_release_non_nested()
3414 		prev_hlock = hlock;  in lock_release_non_nested()
3419 	if (hlock->instance == lock)  in lock_release_non_nested()
3420 		lock_release_holdtime(hlock);  in lock_release_non_nested()
3422 	if (hlock->references) {  in lock_release_non_nested()
3423 		hlock->references--;  in lock_release_non_nested()
3424 		if (hlock->references) {  in lock_release_non_nested()
3441 	curr->curr_chain_key = hlock->prev_chain_key;  in lock_release_non_nested()
3444 		hlock = curr->held_locks + i;  in lock_release_non_nested()
3445 		if (!__lock_acquire(hlock->instance,  in lock_release_non_nested()
3446 			hlock_class(hlock)->subclass, hlock->trylock,  in lock_release_non_nested()
3447 				hlock->read, hlock->check, hlock->hardirqs_off,  in lock_release_non_nested()
3448 				hlock->nest_lock, hlock->acquire_ip,  in lock_release_non_nested()
3449 				hlock->references))  in lock_release_non_nested()
3471 	struct held_lock *hlock;  in lock_release_nested()  local
3478 	hlock = curr->held_locks + depth;  in lock_release_nested()
3483 	if (hlock->instance != lock || hlock->references)  in lock_release_nested()
3490 	if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0)))  in lock_release_nested()
3493 	curr->curr_chain_key = hlock->prev_chain_key;  in lock_release_nested()
3495 	lock_release_holdtime(hlock);  in lock_release_nested()
3498 	hlock->prev_chain_key = 0;  in lock_release_nested()
3499 	hlock->class_idx = 0;  in lock_release_nested()
3500 	hlock->acquire_ip = 0;  in lock_release_nested()
3501 	hlock->irq_context = 0;  in lock_release_nested()
3537 		struct held_lock *hlock = curr->held_locks + i;  in __lock_is_held()  local
3539 		if (match_held_lock(hlock, lock))  in __lock_is_held()
3712 	struct held_lock *hlock, *prev_hlock;  in __lock_contended()  local
3727 		hlock = curr->held_locks + i;  in __lock_contended()
3731 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_contended()
3733 		if (match_held_lock(hlock, lock))  in __lock_contended()
3735 		prev_hlock = hlock;  in __lock_contended()
3741 	if (hlock->instance != lock)  in __lock_contended()
3744 	hlock->waittime_stamp = lockstat_clock();  in __lock_contended()
3746 	contention_point = lock_point(hlock_class(hlock)->contention_point, ip);  in __lock_contended()
3747 	contending_point = lock_point(hlock_class(hlock)->contending_point,  in __lock_contended()
3750 	stats = get_lock_stats(hlock_class(hlock));  in __lock_contended()
3756 		stats->bounces[bounce_contended + !!hlock->read]++;  in __lock_contended()
3764 	struct held_lock *hlock, *prev_hlock;  in __lock_acquired()  local
3780 		hlock = curr->held_locks + i;  in __lock_acquired()
3784 		if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)  in __lock_acquired()
3786 		if (match_held_lock(hlock, lock))  in __lock_acquired()
3788 		prev_hlock = hlock;  in __lock_acquired()
3794 	if (hlock->instance != lock)  in __lock_acquired()
3798 	if (hlock->waittime_stamp) {  in __lock_acquired()
3800 		waittime = now - hlock->waittime_stamp;  in __lock_acquired()
3801 		hlock->holdtime_stamp = now;  in __lock_acquired()
3806 	stats = get_lock_stats(hlock_class(hlock));  in __lock_acquired()
3808 		if (hlock->read)  in __lock_acquired()
3814 		stats->bounces[bounce_acquired + !!hlock->read]++;  in __lock_acquired()
4080 		     const void *mem_to, struct held_lock *hlock)  in print_freed_lock_bug()  argument
4094 	print_lock(hlock);  in print_freed_lock_bug()
4116 	struct held_lock *hlock;  in debug_check_no_locks_freed()  local
4125 		hlock = curr->held_locks + i;  in debug_check_no_locks_freed()
4127 		if (not_in_range(mem_from, mem_len, hlock->instance,  in debug_check_no_locks_freed()
4128 					sizeof(*hlock->instance)))  in debug_check_no_locks_freed()
4131 		print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);  in debug_check_no_locks_freed()