200 	struct workqueue_struct *wq;		/* I: the owning workqueue */  member
340 static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
350 #define assert_rcu_or_wq_mutex(wq)					\  argument
352 			   lockdep_is_held(&wq->mutex),			\
355 #define assert_rcu_or_wq_mutex_or_pool_mutex(wq)			\  argument
357 			   lockdep_is_held(&wq->mutex) ||		\
410 #define for_each_pwq(pwq, wq)						\  argument
411 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node)		\
412 		if (({ assert_rcu_or_wq_mutex(wq); false; })) { }	\
572 static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,  in unbound_pwq_by_node()  argument
575 	assert_rcu_or_wq_mutex_or_pool_mutex(wq);  in unbound_pwq_by_node()
584 		return wq->dfl_pwq;  in unbound_pwq_by_node()
586 	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);  in unbound_pwq_by_node()
1093 	if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))  in put_pwq()
1184 	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))  in pwq_dec_nr_in_flight()
1185 		complete(&pwq->wq->first_flusher->done);  in pwq_dec_nr_in_flight()
1329 static bool is_chained_work(struct workqueue_struct *wq)  in is_chained_work()  argument
1338 	return worker && worker->current_pwq->wq == wq;  in is_chained_work()
1341 static void __queue_work(int cpu, struct workqueue_struct *wq,  in __queue_work()  argument
1361 	if (unlikely(wq->flags & __WQ_DRAINING) &&  in __queue_work()
1362 	    WARN_ON_ONCE(!is_chained_work(wq)))  in __queue_work()
1369 	if (!(wq->flags & WQ_UNBOUND))  in __queue_work()
1370 		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);  in __queue_work()
1372 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));  in __queue_work()
1387 		if (worker && worker->current_pwq->wq == wq) {  in __queue_work()
1407 		if (wq->flags & WQ_UNBOUND) {  in __queue_work()
1414 			  wq->name, cpu);  in __queue_work()
1453 bool queue_work_on(int cpu, struct workqueue_struct *wq,  in queue_work_on()  argument
1462 		__queue_work(cpu, wq, work);  in queue_work_on()
1476 	__queue_work(dwork->cpu, dwork->wq, &dwork->work);  in delayed_work_timer_fn()
1480 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,  in __queue_delayed_work()  argument
1498 		__queue_work(cpu, wq, &dwork->work);  in __queue_delayed_work()
1504 	dwork->wq = wq;  in __queue_delayed_work()
1525 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,  in queue_delayed_work_on()  argument
1536 		__queue_delayed_work(cpu, wq, dwork, delay);  in queue_delayed_work_on()
1563 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,  in mod_delayed_work_on()  argument
1574 		__queue_delayed_work(cpu, wq, dwork, delay);  in mod_delayed_work_on()
1842 	struct workqueue_struct *wq = pwq->wq;  in send_mayday()  local
1846 	if (!wq->rescuer)  in send_mayday()
1857 		list_add_tail(&pwq->mayday_node, &wq->maydays);  in send_mayday()
1858 		wake_up_process(wq->rescuer->task);  in send_mayday()
2003 	bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;  in process_one_work()
2073 	lock_map_acquire_read(&pwq->wq->lockdep_map);  in process_one_work()
2083 	lock_map_release(&pwq->wq->lockdep_map);  in process_one_work()
2257 	struct workqueue_struct *wq = rescuer->rescue_wq;  in rescuer_thread()  local
2284 	while (!list_empty(&wq->maydays)) {  in rescuer_thread()
2285 		struct pool_workqueue *pwq = list_first_entry(&wq->maydays,  in rescuer_thread()
2324 				list_move_tail(&pwq->mayday_node, &wq->maydays);  in rescuer_thread()
2470 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,  in flush_workqueue_prep_pwqs()  argument
2477 		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));  in flush_workqueue_prep_pwqs()
2478 		atomic_set(&wq->nr_pwqs_to_flush, 1);  in flush_workqueue_prep_pwqs()
2481 	for_each_pwq(pwq, wq) {  in flush_workqueue_prep_pwqs()
2491 				atomic_inc(&wq->nr_pwqs_to_flush);  in flush_workqueue_prep_pwqs()
2504 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))  in flush_workqueue_prep_pwqs()
2505 		complete(&wq->first_flusher->done);  in flush_workqueue_prep_pwqs()
2517 void flush_workqueue(struct workqueue_struct *wq)  in flush_workqueue()  argument
2526 	lock_map_acquire(&wq->lockdep_map);  in flush_workqueue()
2527 	lock_map_release(&wq->lockdep_map);  in flush_workqueue()
2529 	mutex_lock(&wq->mutex);  in flush_workqueue()
2534 	next_color = work_next_color(wq->work_color);  in flush_workqueue()
2536 	if (next_color != wq->flush_color) {  in flush_workqueue()
2542 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));  in flush_workqueue()
2543 		this_flusher.flush_color = wq->work_color;  in flush_workqueue()
2544 		wq->work_color = next_color;  in flush_workqueue()
2546 		if (!wq->first_flusher) {  in flush_workqueue()
2548 			WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);  in flush_workqueue()
2550 			wq->first_flusher = &this_flusher;  in flush_workqueue()
2552 			if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,  in flush_workqueue()
2553 						       wq->work_color)) {  in flush_workqueue()
2555 				wq->flush_color = next_color;  in flush_workqueue()
2556 				wq->first_flusher = NULL;  in flush_workqueue()
2561 			WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);  in flush_workqueue()
2562 			list_add_tail(&this_flusher.list, &wq->flusher_queue);  in flush_workqueue()
2563 			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);  in flush_workqueue()
2571 		list_add_tail(&this_flusher.list, &wq->flusher_overflow);  in flush_workqueue()
2574 	mutex_unlock(&wq->mutex);  in flush_workqueue()
2584 	if (wq->first_flusher != &this_flusher)  in flush_workqueue()
2587 	mutex_lock(&wq->mutex);  in flush_workqueue()
2590 	if (wq->first_flusher != &this_flusher)  in flush_workqueue()
2593 	wq->first_flusher = NULL;  in flush_workqueue()
2596 	WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);  in flush_workqueue()
2602 		list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {  in flush_workqueue()
2603 			if (next->flush_color != wq->flush_color)  in flush_workqueue()
2609 		WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&  in flush_workqueue()
2610 			     wq->flush_color != work_next_color(wq->work_color));  in flush_workqueue()
2613 		wq->flush_color = work_next_color(wq->flush_color);  in flush_workqueue()
2616 		if (!list_empty(&wq->flusher_overflow)) {  in flush_workqueue()
2623 			list_for_each_entry(tmp, &wq->flusher_overflow, list)  in flush_workqueue()
2624 				tmp->flush_color = wq->work_color;  in flush_workqueue()
2626 			wq->work_color = work_next_color(wq->work_color);  in flush_workqueue()
2628 			list_splice_tail_init(&wq->flusher_overflow,  in flush_workqueue()
2629 					      &wq->flusher_queue);  in flush_workqueue()
2630 			flush_workqueue_prep_pwqs(wq, -1, wq->work_color);  in flush_workqueue()
2633 		if (list_empty(&wq->flusher_queue)) {  in flush_workqueue()
2634 			WARN_ON_ONCE(wq->flush_color != wq->work_color);  in flush_workqueue()
2642 		WARN_ON_ONCE(wq->flush_color == wq->work_color);  in flush_workqueue()
2643 		WARN_ON_ONCE(wq->flush_color != next->flush_color);  in flush_workqueue()
2646 		wq->first_flusher = next;  in flush_workqueue()
2648 		if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))  in flush_workqueue()
2655 		wq->first_flusher = NULL;  in flush_workqueue()
2659 	mutex_unlock(&wq->mutex);  in flush_workqueue()
2674 void drain_workqueue(struct workqueue_struct *wq)  in drain_workqueue()  argument
2684 	mutex_lock(&wq->mutex);  in drain_workqueue()
2685 	if (!wq->nr_drainers++)  in drain_workqueue()
2686 		wq->flags |= __WQ_DRAINING;  in drain_workqueue()
2687 	mutex_unlock(&wq->mutex);  in drain_workqueue()
2689 	flush_workqueue(wq);  in drain_workqueue()
2691 	mutex_lock(&wq->mutex);  in drain_workqueue()
2693 	for_each_pwq(pwq, wq) {  in drain_workqueue()
2706 				wq->name, flush_cnt);  in drain_workqueue()
2708 		mutex_unlock(&wq->mutex);  in drain_workqueue()
2712 	if (!--wq->nr_drainers)  in drain_workqueue()
2713 		wq->flags &= ~__WQ_DRAINING;  in drain_workqueue()
2714 	mutex_unlock(&wq->mutex);  in drain_workqueue()
2755 	if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)  in start_flush_work()
2756 		lock_map_acquire(&pwq->wq->lockdep_map);  in start_flush_work()
2758 		lock_map_acquire_read(&pwq->wq->lockdep_map);  in start_flush_work()
2759 	lock_map_release(&pwq->wq->lockdep_map);  in start_flush_work()
2907 		__queue_work(dwork->cpu, dwork->wq, &dwork->work);  in flush_delayed_work()
3176 	struct workqueue_struct *wq =  in rcu_free_wq()  local
3179 	if (!(wq->flags & WQ_UNBOUND))  in rcu_free_wq()
3180 		free_percpu(wq->cpu_pwqs);  in rcu_free_wq()
3182 		free_workqueue_attrs(wq->unbound_attrs);  in rcu_free_wq()
3184 	kfree(wq->rescuer);  in rcu_free_wq()
3185 	kfree(wq);  in rcu_free_wq()
3345 	struct workqueue_struct *wq = pwq->wq;  in pwq_unbound_release_workfn()  local
3349 	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))  in pwq_unbound_release_workfn()
3352 	mutex_lock(&wq->mutex);  in pwq_unbound_release_workfn()
3354 	is_last = list_empty(&wq->pwqs);  in pwq_unbound_release_workfn()
3355 	mutex_unlock(&wq->mutex);  in pwq_unbound_release_workfn()
3368 		call_rcu_sched(&wq->rcu, rcu_free_wq);  in pwq_unbound_release_workfn()
3381 	struct workqueue_struct *wq = pwq->wq;  in pwq_adjust_max_active()  local
3382 	bool freezable = wq->flags & WQ_FREEZABLE;  in pwq_adjust_max_active()
3385 	lockdep_assert_held(&wq->mutex);  in pwq_adjust_max_active()
3388 	if (!freezable && pwq->max_active == wq->saved_max_active)  in pwq_adjust_max_active()
3399 		pwq->max_active = wq->saved_max_active;  in pwq_adjust_max_active()
3418 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,  in init_pwq()  argument
3426 	pwq->wq = wq;  in init_pwq()
3438 	struct workqueue_struct *wq = pwq->wq;  in link_pwq()  local
3440 	lockdep_assert_held(&wq->mutex);  in link_pwq()
3447 	pwq->work_color = wq->work_color;  in link_pwq()
3453 	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);  in link_pwq()
3457 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,  in alloc_unbound_pwq()  argument
3475 	init_pwq(pwq, wq, pool);  in alloc_unbound_pwq()
3525 static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,  in numa_pwq_tbl_install()  argument
3532 	lockdep_assert_held(&wq->mutex);  in numa_pwq_tbl_install()
3537 	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);  in numa_pwq_tbl_install()
3538 	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);  in numa_pwq_tbl_install()
3544 	struct workqueue_struct	*wq;		/* target workqueue */  member
3568 apply_wqattrs_prepare(struct workqueue_struct *wq,  in apply_wqattrs_prepare()  argument
3601 	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);  in apply_wqattrs_prepare()
3607 			ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);  in apply_wqattrs_prepare()
3617 	ctx->wq = wq;  in apply_wqattrs_prepare()
3634 	mutex_lock(&ctx->wq->mutex);  in apply_wqattrs_commit()
3636 	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);  in apply_wqattrs_commit()
3640 		ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,  in apply_wqattrs_commit()
3645 	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);  in apply_wqattrs_commit()
3647 	mutex_unlock(&ctx->wq->mutex);  in apply_wqattrs_commit()
3666 int apply_workqueue_attrs(struct workqueue_struct *wq,  in apply_workqueue_attrs()  argument
3673 	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))  in apply_workqueue_attrs()
3677 	if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))  in apply_workqueue_attrs()
3688 	ctx = apply_wqattrs_prepare(wq, attrs);  in apply_workqueue_attrs()
3726 static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,  in wq_update_unbound_numa()  argument
3737 	if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND))  in wq_update_unbound_numa()
3748 	mutex_lock(&wq->mutex);  in wq_update_unbound_numa()
3749 	if (wq->unbound_attrs->no_numa)  in wq_update_unbound_numa()
3752 	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);  in wq_update_unbound_numa()
3753 	pwq = unbound_pwq_by_node(wq, node);  in wq_update_unbound_numa()
3761 	if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {  in wq_update_unbound_numa()
3768 	mutex_unlock(&wq->mutex);  in wq_update_unbound_numa()
3771 	pwq = alloc_unbound_pwq(wq, target_attrs);  in wq_update_unbound_numa()
3774 			wq->name);  in wq_update_unbound_numa()
3775 		mutex_lock(&wq->mutex);  in wq_update_unbound_numa()
3785 	mutex_lock(&wq->mutex);  in wq_update_unbound_numa()
3786 	old_pwq = numa_pwq_tbl_install(wq, node, pwq);  in wq_update_unbound_numa()
3790 	spin_lock_irq(&wq->dfl_pwq->pool->lock);  in wq_update_unbound_numa()
3791 	get_pwq(wq->dfl_pwq);  in wq_update_unbound_numa()
3792 	spin_unlock_irq(&wq->dfl_pwq->pool->lock);  in wq_update_unbound_numa()
3793 	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);  in wq_update_unbound_numa()
3795 	mutex_unlock(&wq->mutex);  in wq_update_unbound_numa()
3799 static int alloc_and_link_pwqs(struct workqueue_struct *wq)  in alloc_and_link_pwqs()  argument
3801 	bool highpri = wq->flags & WQ_HIGHPRI;  in alloc_and_link_pwqs()
3804 	if (!(wq->flags & WQ_UNBOUND)) {  in alloc_and_link_pwqs()
3805 		wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);  in alloc_and_link_pwqs()
3806 		if (!wq->cpu_pwqs)  in alloc_and_link_pwqs()
3811 				per_cpu_ptr(wq->cpu_pwqs, cpu);  in alloc_and_link_pwqs()
3815 			init_pwq(pwq, wq, &cpu_pools[highpri]);  in alloc_and_link_pwqs()
3817 			mutex_lock(&wq->mutex);  in alloc_and_link_pwqs()
3819 			mutex_unlock(&wq->mutex);  in alloc_and_link_pwqs()
3822 	} else if (wq->flags & __WQ_ORDERED) {  in alloc_and_link_pwqs()
3823 		ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);  in alloc_and_link_pwqs()
3825 		WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||  in alloc_and_link_pwqs()
3826 			      wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),  in alloc_and_link_pwqs()
3827 		     "ordering guarantee broken for workqueue %s\n", wq->name);  in alloc_and_link_pwqs()
3830 		return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);  in alloc_and_link_pwqs()
3854 	struct workqueue_struct *wq;  in __alloc_workqueue_key()  local
3863 		tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);  in __alloc_workqueue_key()
3865 	wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);  in __alloc_workqueue_key()
3866 	if (!wq)  in __alloc_workqueue_key()
3870 		wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);  in __alloc_workqueue_key()
3871 		if (!wq->unbound_attrs)  in __alloc_workqueue_key()
3876 	vsnprintf(wq->name, sizeof(wq->name), fmt, args);  in __alloc_workqueue_key()
3880 	max_active = wq_clamp_max_active(max_active, flags, wq->name);  in __alloc_workqueue_key()
3883 	wq->flags = flags;  in __alloc_workqueue_key()
3884 	wq->saved_max_active = max_active;  in __alloc_workqueue_key()
3885 	mutex_init(&wq->mutex);  in __alloc_workqueue_key()
3886 	atomic_set(&wq->nr_pwqs_to_flush, 0);  in __alloc_workqueue_key()
3887 	INIT_LIST_HEAD(&wq->pwqs);  in __alloc_workqueue_key()
3888 	INIT_LIST_HEAD(&wq->flusher_queue);  in __alloc_workqueue_key()
3889 	INIT_LIST_HEAD(&wq->flusher_overflow);  in __alloc_workqueue_key()
3890 	INIT_LIST_HEAD(&wq->maydays);  in __alloc_workqueue_key()
3892 	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);  in __alloc_workqueue_key()
3893 	INIT_LIST_HEAD(&wq->list);  in __alloc_workqueue_key()
3895 	if (alloc_and_link_pwqs(wq) < 0)  in __alloc_workqueue_key()
3909 		rescuer->rescue_wq = wq;  in __alloc_workqueue_key()
3911 					       wq->name);  in __alloc_workqueue_key()
3917 		wq->rescuer = rescuer;  in __alloc_workqueue_key()
3922 	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))  in __alloc_workqueue_key()
3932 	mutex_lock(&wq->mutex);  in __alloc_workqueue_key()
3933 	for_each_pwq(pwq, wq)  in __alloc_workqueue_key()
3935 	mutex_unlock(&wq->mutex);  in __alloc_workqueue_key()
3937 	list_add_tail_rcu(&wq->list, &workqueues);  in __alloc_workqueue_key()
3941 	return wq;  in __alloc_workqueue_key()
3944 	free_workqueue_attrs(wq->unbound_attrs);  in __alloc_workqueue_key()
3945 	kfree(wq);  in __alloc_workqueue_key()
3948 	destroy_workqueue(wq);  in __alloc_workqueue_key()
3959 void destroy_workqueue(struct workqueue_struct *wq)  in destroy_workqueue()  argument
3965 	drain_workqueue(wq);  in destroy_workqueue()
3968 	mutex_lock(&wq->mutex);  in destroy_workqueue()
3969 	for_each_pwq(pwq, wq) {  in destroy_workqueue()
3974 				mutex_unlock(&wq->mutex);  in destroy_workqueue()
3979 		if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||  in destroy_workqueue()
3982 			mutex_unlock(&wq->mutex);  in destroy_workqueue()
3986 	mutex_unlock(&wq->mutex);  in destroy_workqueue()
3993 	list_del_rcu(&wq->list);  in destroy_workqueue()
3996 	workqueue_sysfs_unregister(wq);  in destroy_workqueue()
3998 	if (wq->rescuer)  in destroy_workqueue()
3999 		kthread_stop(wq->rescuer->task);  in destroy_workqueue()
4001 	if (!(wq->flags & WQ_UNBOUND)) {  in destroy_workqueue()
4006 		call_rcu_sched(&wq->rcu, rcu_free_wq);  in destroy_workqueue()
4014 			pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);  in destroy_workqueue()
4015 			RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);  in destroy_workqueue()
4023 		pwq = wq->dfl_pwq;  in destroy_workqueue()
4024 		wq->dfl_pwq = NULL;  in destroy_workqueue()
4040 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)  in workqueue_set_max_active()  argument
4045 	if (WARN_ON(wq->flags & __WQ_ORDERED))  in workqueue_set_max_active()
4048 	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);  in workqueue_set_max_active()
4050 	mutex_lock(&wq->mutex);  in workqueue_set_max_active()
4052 	wq->saved_max_active = max_active;  in workqueue_set_max_active()
4054 	for_each_pwq(pwq, wq)  in workqueue_set_max_active()
4057 	mutex_unlock(&wq->mutex);  in workqueue_set_max_active()
4094 bool workqueue_congested(int cpu, struct workqueue_struct *wq)  in workqueue_congested()  argument
4104 	if (!(wq->flags & WQ_UNBOUND))  in workqueue_congested()
4105 		pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);  in workqueue_congested()
4107 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));  in workqueue_congested()
4192 	struct workqueue_struct *wq = NULL;  in print_worker_info()  local
4211 	probe_kernel_read(&wq, &pwq->wq, sizeof(wq));  in print_worker_info()
4212 	probe_kernel_read(name, wq->name, sizeof(name) - 1);  in print_worker_info()
4279 				worker == pwq->wq->rescuer ? "(RESCUER)" : "",  in show_pwq()
4328 	struct workqueue_struct *wq;  in show_workqueue_state()  local
4337 	list_for_each_entry_rcu(wq, &workqueues, list) {  in show_workqueue_state()
4341 		for_each_pwq(pwq, wq) {  in show_workqueue_state()
4350 		pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);  in show_workqueue_state()
4352 		for_each_pwq(pwq, wq) {  in show_workqueue_state()
4573 	struct workqueue_struct *wq;  in workqueue_cpu_up_callback()  local
4602 		list_for_each_entry(wq, &workqueues, list)  in workqueue_cpu_up_callback()
4603 			wq_update_unbound_numa(wq, cpu, true);  in workqueue_cpu_up_callback()
4621 	struct workqueue_struct *wq;  in workqueue_cpu_down_callback()  local
4631 		list_for_each_entry(wq, &workqueues, list)  in workqueue_cpu_down_callback()
4632 			wq_update_unbound_numa(wq, cpu, false);  in workqueue_cpu_down_callback()
4697 	struct workqueue_struct *wq;  in freeze_workqueues_begin()  local
4705 	list_for_each_entry(wq, &workqueues, list) {  in freeze_workqueues_begin()
4706 		mutex_lock(&wq->mutex);  in freeze_workqueues_begin()
4707 		for_each_pwq(pwq, wq)  in freeze_workqueues_begin()
4709 		mutex_unlock(&wq->mutex);  in freeze_workqueues_begin()
4731 	struct workqueue_struct *wq;  in freeze_workqueues_busy()  local
4738 	list_for_each_entry(wq, &workqueues, list) {  in freeze_workqueues_busy()
4739 		if (!(wq->flags & WQ_FREEZABLE))  in freeze_workqueues_busy()
4746 		for_each_pwq(pwq, wq) {  in freeze_workqueues_busy()
4772 	struct workqueue_struct *wq;  in thaw_workqueues()  local
4783 	list_for_each_entry(wq, &workqueues, list) {  in thaw_workqueues()
4784 		mutex_lock(&wq->mutex);  in thaw_workqueues()
4785 		for_each_pwq(pwq, wq)  in thaw_workqueues()
4787 		mutex_unlock(&wq->mutex);  in thaw_workqueues()
4811 	struct workqueue_struct		*wq;  member
4819 	return wq_dev->wq;  in dev_to_wq()
4825 	struct workqueue_struct *wq = dev_to_wq(dev);  in per_cpu_show()  local
4827 	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));  in per_cpu_show()
4834 	struct workqueue_struct *wq = dev_to_wq(dev);  in max_active_show()  local
4836 	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);  in max_active_show()
4843 	struct workqueue_struct *wq = dev_to_wq(dev);  in max_active_store()  local
4849 	workqueue_set_max_active(wq, val);  in max_active_store()
4864 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_pool_ids_show()  local
4872 				     unbound_pwq_by_node(wq, node)->pool->id);  in wq_pool_ids_show()
4884 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_nice_show()  local
4887 	mutex_lock(&wq->mutex);  in wq_nice_show()
4888 	written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);  in wq_nice_show()
4889 	mutex_unlock(&wq->mutex);  in wq_nice_show()
4895 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)  in wq_sysfs_prep_attrs()  argument
4903 	mutex_lock(&wq->mutex);  in wq_sysfs_prep_attrs()
4904 	copy_workqueue_attrs(attrs, wq->unbound_attrs);  in wq_sysfs_prep_attrs()
4905 	mutex_unlock(&wq->mutex);  in wq_sysfs_prep_attrs()
4912 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_nice_store()  local
4916 	attrs = wq_sysfs_prep_attrs(wq);  in wq_nice_store()
4922 		ret = apply_workqueue_attrs(wq, attrs);  in wq_nice_store()
4933 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_cpumask_show()  local
4936 	mutex_lock(&wq->mutex);  in wq_cpumask_show()
4938 			    cpumask_pr_args(wq->unbound_attrs->cpumask));  in wq_cpumask_show()
4939 	mutex_unlock(&wq->mutex);  in wq_cpumask_show()
4947 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_cpumask_store()  local
4951 	attrs = wq_sysfs_prep_attrs(wq);  in wq_cpumask_store()
4957 		ret = apply_workqueue_attrs(wq, attrs);  in wq_cpumask_store()
4966 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_numa_show()  local
4969 	mutex_lock(&wq->mutex);  in wq_numa_show()
4971 			    !wq->unbound_attrs->no_numa);  in wq_numa_show()
4972 	mutex_unlock(&wq->mutex);  in wq_numa_show()
4980 	struct workqueue_struct *wq = dev_to_wq(dev);  in wq_numa_store()  local
4984 	attrs = wq_sysfs_prep_attrs(wq);  in wq_numa_store()
4991 		ret = apply_workqueue_attrs(wq, attrs);  in wq_numa_store()
5039 int workqueue_sysfs_register(struct workqueue_struct *wq)  in workqueue_sysfs_register()  argument
5049 	if (WARN_ON(wq->flags & __WQ_ORDERED))  in workqueue_sysfs_register()
5052 	wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);  in workqueue_sysfs_register()
5056 	wq_dev->wq = wq;  in workqueue_sysfs_register()
5058 	wq_dev->dev.init_name = wq->name;  in workqueue_sysfs_register()
5070 		wq->wq_dev = NULL;  in workqueue_sysfs_register()
5074 	if (wq->flags & WQ_UNBOUND) {  in workqueue_sysfs_register()
5081 				wq->wq_dev = NULL;  in workqueue_sysfs_register()
5098 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)  in workqueue_sysfs_unregister()  argument
5100 	struct wq_device *wq_dev = wq->wq_dev;  in workqueue_sysfs_unregister()
5102 	if (!wq->wq_dev)  in workqueue_sysfs_unregister()
5105 	wq->wq_dev = NULL;  in workqueue_sysfs_unregister()
5109 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)	{ }  in workqueue_sysfs_unregister()  argument