36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)  in policy_is_inactive()  argument
38 	return cpumask_empty(policy->cpus);  in policy_is_inactive()
41 static bool suitable_policy(struct cpufreq_policy *policy, bool active)  in suitable_policy()  argument
43 	return active == !policy_is_inactive(policy);  in suitable_policy()
47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,  in next_policy()  argument
51 		policy = list_next_entry(policy, policy_list);  in next_policy()
54 		if (&policy->policy_list == &cpufreq_policy_list)  in next_policy()
56 	} while (!suitable_policy(policy, active));  in next_policy()
58 	return policy;  in next_policy()
63 	struct cpufreq_policy *policy;  in first_policy()  local
69 	policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),  in first_policy()
72 	if (!suitable_policy(policy, active))  in first_policy()
73 		policy = next_policy(policy, active);  in first_policy()
75 	return policy;  in first_policy()
116 static int __cpufreq_governor(struct cpufreq_policy *policy,
118 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
157 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)  in get_governor_parent_kobj()  argument
160 		return &policy->kobj;  in get_governor_parent_kobj()
168 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_frequency_get_table()  local
170 	return policy && !policy_is_inactive(policy) ?  in cpufreq_frequency_get_table()
171 		policy->freq_table : NULL;  in cpufreq_frequency_get_table()
217 int cpufreq_generic_init(struct cpufreq_policy *policy,  in cpufreq_generic_init()  argument
223 	ret = cpufreq_table_validate_and_show(policy, table);  in cpufreq_generic_init()
229 	policy->cpuinfo.transition_latency = transition_latency;  in cpufreq_generic_init()
235 	cpumask_setall(policy->cpus);  in cpufreq_generic_init()
243 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_cpu_get_raw()  local
245 	return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;  in cpufreq_cpu_get_raw()
251 	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);  in cpufreq_generic_get()  local
253 	if (!policy || IS_ERR(policy->clk)) {  in cpufreq_generic_get()
255 		       __func__, policy ? "clk" : "policy", cpu);  in cpufreq_generic_get()
259 	return clk_get_rate(policy->clk) / 1000;  in cpufreq_generic_get()
278 	struct cpufreq_policy *policy = NULL;  in cpufreq_cpu_get()  local
289 		policy = cpufreq_cpu_get_raw(cpu);  in cpufreq_cpu_get()
290 		if (policy)  in cpufreq_cpu_get()
291 			kobject_get(&policy->kobj);  in cpufreq_cpu_get()
296 	return policy;  in cpufreq_cpu_get()
308 void cpufreq_cpu_put(struct cpufreq_policy *policy)  in cpufreq_cpu_put()  argument
310 	kobject_put(&policy->kobj);  in cpufreq_cpu_put()
350 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,  in __cpufreq_notify_transition()  argument
370 			if ((policy) && (policy->cpu == freqs->cpu) &&  in __cpufreq_notify_transition()
371 			    (policy->cur) && (policy->cur != freqs->old)) {  in __cpufreq_notify_transition()
373 					 freqs->old, policy->cur);  in __cpufreq_notify_transition()
374 				freqs->old = policy->cur;  in __cpufreq_notify_transition()
389 		if (likely(policy) && likely(policy->cpu == freqs->cpu))  in __cpufreq_notify_transition()
390 			policy->cur = freqs->new;  in __cpufreq_notify_transition()
403 static void cpufreq_notify_transition(struct cpufreq_policy *policy,  in cpufreq_notify_transition()  argument
406 	for_each_cpu(freqs->cpu, policy->cpus)  in cpufreq_notify_transition()
407 		__cpufreq_notify_transition(policy, freqs, state);  in cpufreq_notify_transition()
411 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,  in cpufreq_notify_post_transition()  argument
414 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);  in cpufreq_notify_post_transition()
419 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);  in cpufreq_notify_post_transition()
420 	cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);  in cpufreq_notify_post_transition()
423 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,  in cpufreq_freq_transition_begin()  argument
436 				&& current == policy->transition_task);  in cpufreq_freq_transition_begin()
439 	wait_event(policy->transition_wait, !policy->transition_ongoing);  in cpufreq_freq_transition_begin()
441 	spin_lock(&policy->transition_lock);  in cpufreq_freq_transition_begin()
443 	if (unlikely(policy->transition_ongoing)) {  in cpufreq_freq_transition_begin()
444 		spin_unlock(&policy->transition_lock);  in cpufreq_freq_transition_begin()
448 	policy->transition_ongoing = true;  in cpufreq_freq_transition_begin()
449 	policy->transition_task = current;  in cpufreq_freq_transition_begin()
451 	spin_unlock(&policy->transition_lock);  in cpufreq_freq_transition_begin()
453 	cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);  in cpufreq_freq_transition_begin()
457 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,  in cpufreq_freq_transition_end()  argument
460 	if (unlikely(WARN_ON(!policy->transition_ongoing)))  in cpufreq_freq_transition_end()
463 	cpufreq_notify_post_transition(policy, freqs, transition_failed);  in cpufreq_freq_transition_end()
465 	policy->transition_ongoing = false;  in cpufreq_freq_transition_end()
466 	policy->transition_task = NULL;  in cpufreq_freq_transition_end()
468 	wake_up(&policy->transition_wait);  in cpufreq_freq_transition_end()
518 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,  in cpufreq_parse_governor()  argument
525 			*policy = CPUFREQ_POLICY_PERFORMANCE;  in cpufreq_parse_governor()
529 			*policy = CPUFREQ_POLICY_POWERSAVE;  in cpufreq_parse_governor()
570 (struct cpufreq_policy *policy, char *buf)		\
572 	return sprintf(buf, "%u\n", policy->object);	\
581 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)  in show_scaling_cur_freq()  argument
586 		ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));  in show_scaling_cur_freq()
588 		ret = sprintf(buf, "%u\n", policy->cur);  in show_scaling_cur_freq()
592 static int cpufreq_set_policy(struct cpufreq_policy *policy,
600 (struct cpufreq_policy *policy, const char *buf, size_t count)		\
605 	memcpy(&new_policy, policy, sizeof(*policy));			\
612 	ret = cpufreq_set_policy(policy, &new_policy);		\
614 		policy->user_policy.object = temp;			\
625 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,  in show_cpuinfo_cur_freq()  argument
628 	unsigned int cur_freq = __cpufreq_get(policy);  in show_cpuinfo_cur_freq()
637 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)  in show_scaling_governor()  argument
639 	if (policy->policy == CPUFREQ_POLICY_POWERSAVE)  in show_scaling_governor()
641 	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)  in show_scaling_governor()
643 	else if (policy->governor)  in show_scaling_governor()
645 				policy->governor->name);  in show_scaling_governor()
652 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,  in store_scaling_governor()  argument
659 	memcpy(&new_policy, policy, sizeof(*policy));  in store_scaling_governor()
665 	if (cpufreq_parse_governor(str_governor, &new_policy.policy,  in store_scaling_governor()
669 	ret = cpufreq_set_policy(policy, &new_policy);  in store_scaling_governor()
676 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)  in show_scaling_driver()  argument
684 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,  in show_scaling_available_governors()  argument
727 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)  in show_related_cpus()  argument
729 	return cpufreq_show_cpus(policy->related_cpus, buf);  in show_related_cpus()
735 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)  in show_affected_cpus()  argument
737 	return cpufreq_show_cpus(policy->cpus, buf);  in show_affected_cpus()
740 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,  in store_scaling_setspeed()  argument
746 	if (!policy->governor || !policy->governor->store_setspeed)  in store_scaling_setspeed()
753 	policy->governor->store_setspeed(policy, freq);  in store_scaling_setspeed()
758 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)  in show_scaling_setspeed()  argument
760 	if (!policy->governor || !policy->governor->show_setspeed)  in show_scaling_setspeed()
763 	return policy->governor->show_setspeed(policy, buf);  in show_scaling_setspeed()
769 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)  in show_bios_limit()  argument
774 		ret = cpufreq_driver->bios_limit(policy->cpu, &limit);  in show_bios_limit()
778 	return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);  in show_bios_limit()
816 	struct cpufreq_policy *policy = to_policy(kobj);  in show()  local
820 	down_read(&policy->rwsem);  in show()
823 		ret = fattr->show(policy, buf);  in show()
827 	up_read(&policy->rwsem);  in show()
835 	struct cpufreq_policy *policy = to_policy(kobj);  in store()  local
841 	if (!cpu_online(policy->cpu))  in store()
844 	down_write(&policy->rwsem);  in store()
847 		ret = fattr->store(policy, buf, count);  in store()
851 	up_write(&policy->rwsem);  in store()
860 	struct cpufreq_policy *policy = to_policy(kobj);  in cpufreq_sysfs_release()  local
862 	complete(&policy->kobj_unregister);  in cpufreq_sysfs_release()
876 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)  in add_cpu_dev_symlink()  argument
882 	if (!policy)  in add_cpu_dev_symlink()
889 	return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");  in add_cpu_dev_symlink()
892 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)  in remove_cpu_dev_symlink()  argument
906 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)  in cpufreq_add_dev_symlink()  argument
912 	for_each_cpu(j, policy->real_cpus) {  in cpufreq_add_dev_symlink()
913 		ret = add_cpu_dev_symlink(policy, j);  in cpufreq_add_dev_symlink()
921 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)  in cpufreq_remove_dev_symlink()  argument
926 	for_each_cpu(j, policy->real_cpus)  in cpufreq_remove_dev_symlink()
927 		remove_cpu_dev_symlink(policy, j);  in cpufreq_remove_dev_symlink()
930 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)  in cpufreq_add_dev_interface()  argument
938 		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));  in cpufreq_add_dev_interface()
944 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);  in cpufreq_add_dev_interface()
949 	ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);  in cpufreq_add_dev_interface()
954 		ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);  in cpufreq_add_dev_interface()
959 	return cpufreq_add_dev_symlink(policy);  in cpufreq_add_dev_interface()
962 static int cpufreq_init_policy(struct cpufreq_policy *policy)  in cpufreq_init_policy()  argument
967 	memcpy(&new_policy, policy, sizeof(*policy));  in cpufreq_init_policy()
970 	gov = find_governor(policy->last_governor);  in cpufreq_init_policy()
973 				policy->governor->name, policy->cpu);  in cpufreq_init_policy()
981 		if (policy->last_policy)  in cpufreq_init_policy()
982 			new_policy.policy = policy->last_policy;  in cpufreq_init_policy()
984 			cpufreq_parse_governor(gov->name, &new_policy.policy,  in cpufreq_init_policy()
988 	return cpufreq_set_policy(policy, &new_policy);  in cpufreq_init_policy()
991 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)  in cpufreq_add_policy_cpu()  argument
996 	if (cpumask_test_cpu(cpu, policy->cpus))  in cpufreq_add_policy_cpu()
1000 		ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);  in cpufreq_add_policy_cpu()
1007 	down_write(&policy->rwsem);  in cpufreq_add_policy_cpu()
1008 	cpumask_set_cpu(cpu, policy->cpus);  in cpufreq_add_policy_cpu()
1009 	up_write(&policy->rwsem);  in cpufreq_add_policy_cpu()
1012 		ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);  in cpufreq_add_policy_cpu()
1014 			ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);  in cpufreq_add_policy_cpu()
1028 	struct cpufreq_policy *policy;  in cpufreq_policy_alloc()  local
1033 	policy = kzalloc(sizeof(*policy), GFP_KERNEL);  in cpufreq_policy_alloc()
1034 	if (!policy)  in cpufreq_policy_alloc()
1037 	if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))  in cpufreq_policy_alloc()
1040 	if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))  in cpufreq_policy_alloc()
1043 	if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))  in cpufreq_policy_alloc()
1046 	kobject_init(&policy->kobj, &ktype_cpufreq);  in cpufreq_policy_alloc()
1047 	INIT_LIST_HEAD(&policy->policy_list);  in cpufreq_policy_alloc()
1048 	init_rwsem(&policy->rwsem);  in cpufreq_policy_alloc()
1049 	spin_lock_init(&policy->transition_lock);  in cpufreq_policy_alloc()
1050 	init_waitqueue_head(&policy->transition_wait);  in cpufreq_policy_alloc()
1051 	init_completion(&policy->kobj_unregister);  in cpufreq_policy_alloc()
1052 	INIT_WORK(&policy->update, handle_update);  in cpufreq_policy_alloc()
1054 	policy->cpu = cpu;  in cpufreq_policy_alloc()
1055 	return policy;  in cpufreq_policy_alloc()
1058 	free_cpumask_var(policy->related_cpus);  in cpufreq_policy_alloc()
1060 	free_cpumask_var(policy->cpus);  in cpufreq_policy_alloc()
1062 	kfree(policy);  in cpufreq_policy_alloc()
1067 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)  in cpufreq_policy_put_kobj()  argument
1074 					     CPUFREQ_REMOVE_POLICY, policy);  in cpufreq_policy_put_kobj()
1076 	down_write(&policy->rwsem);  in cpufreq_policy_put_kobj()
1077 	cpufreq_remove_dev_symlink(policy);  in cpufreq_policy_put_kobj()
1078 	kobj = &policy->kobj;  in cpufreq_policy_put_kobj()
1079 	cmp = &policy->kobj_unregister;  in cpufreq_policy_put_kobj()
1080 	up_write(&policy->rwsem);  in cpufreq_policy_put_kobj()
1093 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)  in cpufreq_policy_free()  argument
1100 	list_del(&policy->policy_list);  in cpufreq_policy_free()
1102 	for_each_cpu(cpu, policy->related_cpus)  in cpufreq_policy_free()
1106 	cpufreq_policy_put_kobj(policy, notify);  in cpufreq_policy_free()
1107 	free_cpumask_var(policy->real_cpus);  in cpufreq_policy_free()
1108 	free_cpumask_var(policy->related_cpus);  in cpufreq_policy_free()
1109 	free_cpumask_var(policy->cpus);  in cpufreq_policy_free()
1110 	kfree(policy);  in cpufreq_policy_free()
1115 	struct cpufreq_policy *policy;  in cpufreq_online()  local
1124 	policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_online()
1125 	if (policy) {  in cpufreq_online()
1126 		WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));  in cpufreq_online()
1127 		if (!policy_is_inactive(policy))  in cpufreq_online()
1128 			return cpufreq_add_policy_cpu(policy, cpu);  in cpufreq_online()
1132 		down_write(&policy->rwsem);  in cpufreq_online()
1133 		policy->cpu = cpu;  in cpufreq_online()
1134 		policy->governor = NULL;  in cpufreq_online()
1135 		up_write(&policy->rwsem);  in cpufreq_online()
1138 		policy = cpufreq_policy_alloc(cpu);  in cpufreq_online()
1139 		if (!policy)  in cpufreq_online()
1143 	cpumask_copy(policy->cpus, cpumask_of(cpu));  in cpufreq_online()
1148 	ret = cpufreq_driver->init(policy);  in cpufreq_online()
1154 	down_write(&policy->rwsem);  in cpufreq_online()
1158 		cpumask_copy(policy->related_cpus, policy->cpus);  in cpufreq_online()
1160 		cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask);  in cpufreq_online()
1163 		ret = kobject_add(&policy->kobj, cpufreq_global_kobject,  in cpufreq_online()
1165 				  cpumask_first(policy->related_cpus));  in cpufreq_online()
1177 	cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);  in cpufreq_online()
1180 		policy->user_policy.min = policy->min;  in cpufreq_online()
1181 		policy->user_policy.max = policy->max;  in cpufreq_online()
1184 		for_each_cpu(j, policy->related_cpus)  in cpufreq_online()
1185 			per_cpu(cpufreq_cpu_data, j) = policy;  in cpufreq_online()
1190 		policy->cur = cpufreq_driver->get(policy->cpu);  in cpufreq_online()
1191 		if (!policy->cur) {  in cpufreq_online()
1218 		ret = cpufreq_frequency_table_get_index(policy, policy->cur);  in cpufreq_online()
1222 				__func__, policy->cpu, policy->cur);  in cpufreq_online()
1223 			ret = __cpufreq_driver_target(policy, policy->cur - 1,  in cpufreq_online()
1233 				__func__, policy->cpu, policy->cur);  in cpufreq_online()
1238 				     CPUFREQ_START, policy);  in cpufreq_online()
1241 		ret = cpufreq_add_dev_interface(policy);  in cpufreq_online()
1245 				CPUFREQ_CREATE_POLICY, policy);  in cpufreq_online()
1248 		list_add(&policy->policy_list, &cpufreq_policy_list);  in cpufreq_online()
1252 	ret = cpufreq_init_policy(policy);  in cpufreq_online()
1261 	up_write(&policy->rwsem);  in cpufreq_online()
1263 	kobject_uevent(&policy->kobj, KOBJ_ADD);  in cpufreq_online()
1267 		cpufreq_driver->ready(policy);  in cpufreq_online()
1274 	up_write(&policy->rwsem);  in cpufreq_online()
1277 		cpufreq_driver->exit(policy);  in cpufreq_online()
1279 	cpufreq_policy_free(policy, !new_policy);  in cpufreq_online()
1303 		struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_add_dev()  local
1305 		ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus)  in cpufreq_add_dev()
1306 			? add_cpu_dev_symlink(policy, cpu) : 0;  in cpufreq_add_dev()
1314 	struct cpufreq_policy *policy;  in cpufreq_offline_prepare()  local
1318 	policy = cpufreq_cpu_get_raw(cpu);  in cpufreq_offline_prepare()
1319 	if (!policy) {  in cpufreq_offline_prepare()
1325 		int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);  in cpufreq_offline_prepare()
1330 	down_write(&policy->rwsem);  in cpufreq_offline_prepare()
1331 	cpumask_clear_cpu(cpu, policy->cpus);  in cpufreq_offline_prepare()
1333 	if (policy_is_inactive(policy)) {  in cpufreq_offline_prepare()
1335 			strncpy(policy->last_governor, policy->governor->name,  in cpufreq_offline_prepare()
1338 			policy->last_policy = policy->policy;  in cpufreq_offline_prepare()
1339 	} else if (cpu == policy->cpu) {  in cpufreq_offline_prepare()
1341 		policy->cpu = cpumask_any(policy->cpus);  in cpufreq_offline_prepare()
1343 	up_write(&policy->rwsem);  in cpufreq_offline_prepare()
1346 	if (!policy_is_inactive(policy)) {  in cpufreq_offline_prepare()
1348 			int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);  in cpufreq_offline_prepare()
1350 				ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);  in cpufreq_offline_prepare()
1356 		cpufreq_driver->stop_cpu(policy);  in cpufreq_offline_prepare()
1362 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_offline_finish()  local
1364 	if (!policy) {  in cpufreq_offline_finish()
1370 	if (!policy_is_inactive(policy))  in cpufreq_offline_finish()
1375 		int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);  in cpufreq_offline_finish()
1386 		cpufreq_driver->exit(policy);  in cpufreq_offline_finish()
1387 		policy->freq_table = NULL;  in cpufreq_offline_finish()
1399 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);  in cpufreq_remove_dev()  local
1401 	if (!policy)  in cpufreq_remove_dev()
1409 	cpumask_clear_cpu(cpu, policy->real_cpus);  in cpufreq_remove_dev()
1410 	remove_cpu_dev_symlink(policy, cpu);  in cpufreq_remove_dev()
1412 	if (cpumask_empty(policy->real_cpus))  in cpufreq_remove_dev()
1413 		cpufreq_policy_free(policy, true);  in cpufreq_remove_dev()
1418 	struct cpufreq_policy *policy =  in handle_update()  local
1420 	unsigned int cpu = policy->cpu;  in handle_update()
1434 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,  in cpufreq_out_of_sync()  argument
1440 		 policy->cur, new_freq);  in cpufreq_out_of_sync()
1442 	freqs.old = policy->cur;  in cpufreq_out_of_sync()
1445 	cpufreq_freq_transition_begin(policy, &freqs);  in cpufreq_out_of_sync()
1446 	cpufreq_freq_transition_end(policy, &freqs, 0);  in cpufreq_out_of_sync()
1458 	struct cpufreq_policy *policy;  in cpufreq_quick_get()  local
1464 	policy = cpufreq_cpu_get(cpu);  in cpufreq_quick_get()
1465 	if (policy) {  in cpufreq_quick_get()
1466 		ret_freq = policy->cur;  in cpufreq_quick_get()
1467 		cpufreq_cpu_put(policy);  in cpufreq_quick_get()
1482 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_quick_get_max()  local
1485 	if (policy) {  in cpufreq_quick_get_max()
1486 		ret_freq = policy->max;  in cpufreq_quick_get_max()
1487 		cpufreq_cpu_put(policy);  in cpufreq_quick_get_max()
1494 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)  in __cpufreq_get()  argument
1501 	ret_freq = cpufreq_driver->get(policy->cpu);  in __cpufreq_get()
1504 	if (unlikely(policy_is_inactive(policy)))  in __cpufreq_get()
1507 	if (ret_freq && policy->cur &&  in __cpufreq_get()
1511 		if (unlikely(ret_freq != policy->cur)) {  in __cpufreq_get()
1512 			cpufreq_out_of_sync(policy, ret_freq);  in __cpufreq_get()
1513 			schedule_work(&policy->update);  in __cpufreq_get()
1528 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_get()  local
1531 	if (policy) {  in cpufreq_get()
1532 		down_read(&policy->rwsem);  in cpufreq_get()
1533 		ret_freq = __cpufreq_get(policy);  in cpufreq_get()
1534 		up_read(&policy->rwsem);  in cpufreq_get()
1536 		cpufreq_cpu_put(policy);  in cpufreq_get()
1554 int cpufreq_generic_suspend(struct cpufreq_policy *policy)  in cpufreq_generic_suspend()  argument
1558 	if (!policy->suspend_freq) {  in cpufreq_generic_suspend()
1564 			policy->suspend_freq);  in cpufreq_generic_suspend()
1566 	ret = __cpufreq_driver_target(policy, policy->suspend_freq,  in cpufreq_generic_suspend()
1570 				__func__, policy->suspend_freq, ret);  in cpufreq_generic_suspend()
1586 	struct cpufreq_policy *policy;  in cpufreq_suspend()  local
1596 	for_each_active_policy(policy) {  in cpufreq_suspend()
1597 		if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))  in cpufreq_suspend()
1599 				__func__, policy);  in cpufreq_suspend()
1601 		    && cpufreq_driver->suspend(policy))  in cpufreq_suspend()
1603 				policy);  in cpufreq_suspend()
1618 	struct cpufreq_policy *policy;  in cpufreq_resume()  local
1630 	for_each_active_policy(policy) {  in cpufreq_resume()
1631 		if (cpufreq_driver->resume && cpufreq_driver->resume(policy))  in cpufreq_resume()
1633 				policy);  in cpufreq_resume()
1634 		else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)  in cpufreq_resume()
1635 		    || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))  in cpufreq_resume()
1637 				__func__, policy);  in cpufreq_resume()
1645 	policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));  in cpufreq_resume()
1646 	if (WARN_ON(!policy))  in cpufreq_resume()
1649 	schedule_work(&policy->update);  in cpufreq_resume()
1765 static int __target_intermediate(struct cpufreq_policy *policy,  in __target_intermediate()  argument
1770 	freqs->new = cpufreq_driver->get_intermediate(policy, index);  in __target_intermediate()
1777 		 __func__, policy->cpu, freqs->old, freqs->new);  in __target_intermediate()
1779 	cpufreq_freq_transition_begin(policy, freqs);  in __target_intermediate()
1780 	ret = cpufreq_driver->target_intermediate(policy, index);  in __target_intermediate()
1781 	cpufreq_freq_transition_end(policy, freqs, ret);  in __target_intermediate()
1790 static int __target_index(struct cpufreq_policy *policy,  in __target_index()  argument
1793 	struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};  in __target_index()
1802 			retval = __target_intermediate(policy, &freqs, index);  in __target_index()
1814 			 __func__, policy->cpu, freqs.old, freqs.new);  in __target_index()
1816 		cpufreq_freq_transition_begin(policy, &freqs);  in __target_index()
1819 	retval = cpufreq_driver->target_index(policy, index);  in __target_index()
1825 		cpufreq_freq_transition_end(policy, &freqs, retval);  in __target_index()
1835 			freqs.new = policy->restore_freq;  in __target_index()
1836 			cpufreq_freq_transition_begin(policy, &freqs);  in __target_index()
1837 			cpufreq_freq_transition_end(policy, &freqs, 0);  in __target_index()
1844 int __cpufreq_driver_target(struct cpufreq_policy *policy,  in __cpufreq_driver_target()  argument
1855 	if (target_freq > policy->max)  in __cpufreq_driver_target()
1856 		target_freq = policy->max;  in __cpufreq_driver_target()
1857 	if (target_freq < policy->min)  in __cpufreq_driver_target()
1858 		target_freq = policy->min;  in __cpufreq_driver_target()
1861 		 policy->cpu, target_freq, relation, old_target_freq);  in __cpufreq_driver_target()
1869 	if (target_freq == policy->cur)  in __cpufreq_driver_target()
1873 	policy->restore_freq = policy->cur;  in __cpufreq_driver_target()
1876 		retval = cpufreq_driver->target(policy, target_freq, relation);  in __cpufreq_driver_target()
1881 		freq_table = cpufreq_frequency_get_table(policy->cpu);  in __cpufreq_driver_target()
1887 		retval = cpufreq_frequency_table_target(policy, freq_table,  in __cpufreq_driver_target()
1894 		if (freq_table[index].frequency == policy->cur) {  in __cpufreq_driver_target()
1899 		retval = __target_index(policy, freq_table, index);  in __cpufreq_driver_target()
1907 int cpufreq_driver_target(struct cpufreq_policy *policy,  in cpufreq_driver_target()  argument
1913 	down_write(&policy->rwsem);  in cpufreq_driver_target()
1915 	ret = __cpufreq_driver_target(policy, target_freq, relation);  in cpufreq_driver_target()
1917 	up_write(&policy->rwsem);  in cpufreq_driver_target()
1923 static int __cpufreq_governor(struct cpufreq_policy *policy,  in __cpufreq_governor()  argument
1945 	if (!policy->governor)  in __cpufreq_governor()
1948 	if (policy->governor->max_transition_latency &&  in __cpufreq_governor()
1949 	    policy->cpuinfo.transition_latency >  in __cpufreq_governor()
1950 	    policy->governor->max_transition_latency) {  in __cpufreq_governor()
1955 				policy->governor->name, gov->name);  in __cpufreq_governor()
1956 			policy->governor = gov;  in __cpufreq_governor()
1961 		if (!try_module_get(policy->governor->owner))  in __cpufreq_governor()
1964 	pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event);  in __cpufreq_governor()
1967 	if ((policy->governor_enabled && event == CPUFREQ_GOV_START)  in __cpufreq_governor()
1968 	    || (!policy->governor_enabled  in __cpufreq_governor()
1975 		policy->governor_enabled = false;  in __cpufreq_governor()
1977 		policy->governor_enabled = true;  in __cpufreq_governor()
1981 	ret = policy->governor->governor(policy, event);  in __cpufreq_governor()
1985 			policy->governor->initialized++;  in __cpufreq_governor()
1987 			policy->governor->initialized--;  in __cpufreq_governor()
1992 			policy->governor_enabled = true;  in __cpufreq_governor()
1994 			policy->governor_enabled = false;  in __cpufreq_governor()
2000 		module_put(policy->governor->owner);  in __cpufreq_governor()
2031 	struct cpufreq_policy *policy;  in cpufreq_unregister_governor()  local
2042 	for_each_inactive_policy(policy) {  in cpufreq_unregister_governor()
2043 		if (!strcmp(policy->last_governor, governor->name)) {  in cpufreq_unregister_governor()
2044 			policy->governor = NULL;  in cpufreq_unregister_governor()
2045 			strcpy(policy->last_governor, "\0");  in cpufreq_unregister_governor()
2069 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)  in cpufreq_get_policy()  argument
2072 	if (!policy)  in cpufreq_get_policy()
2079 	memcpy(policy, cpu_policy, sizeof(*policy));  in cpufreq_get_policy()
2090 static int cpufreq_set_policy(struct cpufreq_policy *policy,  in cpufreq_set_policy()  argument
2099 	memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));  in cpufreq_set_policy()
2129 	policy->min = new_policy->min;  in cpufreq_set_policy()
2130 	policy->max = new_policy->max;  in cpufreq_set_policy()
2133 		 policy->min, policy->max);  in cpufreq_set_policy()
2136 		policy->policy = new_policy->policy;  in cpufreq_set_policy()
2141 	if (new_policy->governor == policy->governor)  in cpufreq_set_policy()
2147 	old_gov = policy->governor;  in cpufreq_set_policy()
2150 		ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);  in cpufreq_set_policy()
2158 		up_write(&policy->rwsem);  in cpufreq_set_policy()
2159 		ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);  in cpufreq_set_policy()
2160 		down_write(&policy->rwsem);  in cpufreq_set_policy()
2170 	policy->governor = new_policy->governor;  in cpufreq_set_policy()
2171 	ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);  in cpufreq_set_policy()
2173 		ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);  in cpufreq_set_policy()
2177 		up_write(&policy->rwsem);  in cpufreq_set_policy()
2178 		__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);  in cpufreq_set_policy()
2179 		down_write(&policy->rwsem);  in cpufreq_set_policy()
2183 	pr_debug("starting governor %s failed\n", policy->governor->name);  in cpufreq_set_policy()
2185 		policy->governor = old_gov;  in cpufreq_set_policy()
2186 		if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT))  in cpufreq_set_policy()
2187 			policy->governor = NULL;  in cpufreq_set_policy()
2189 			__cpufreq_governor(policy, CPUFREQ_GOV_START);  in cpufreq_set_policy()
2196 	return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);  in cpufreq_set_policy()
2208 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);  in cpufreq_update_policy()  local
2212 	if (!policy)  in cpufreq_update_policy()
2215 	down_write(&policy->rwsem);  in cpufreq_update_policy()
2218 	memcpy(&new_policy, policy, sizeof(*policy));  in cpufreq_update_policy()
2219 	new_policy.min = policy->user_policy.min;  in cpufreq_update_policy()
2220 	new_policy.max = policy->user_policy.max;  in cpufreq_update_policy()
2233 		if (!policy->cur) {  in cpufreq_update_policy()
2235 			policy->cur = new_policy.cur;  in cpufreq_update_policy()
2237 			if (policy->cur != new_policy.cur && has_target())  in cpufreq_update_policy()
2238 				cpufreq_out_of_sync(policy, new_policy.cur);  in cpufreq_update_policy()
2242 	ret = cpufreq_set_policy(policy, &new_policy);  in cpufreq_update_policy()
2245 	up_write(&policy->rwsem);  in cpufreq_update_policy()
2247 	cpufreq_cpu_put(policy);  in cpufreq_update_policy()
2287 	struct cpufreq_policy *policy;  in cpufreq_boost_set_sw()  local
2290 	for_each_active_policy(policy) {  in cpufreq_boost_set_sw()
2291 		freq_table = cpufreq_frequency_get_table(policy->cpu);  in cpufreq_boost_set_sw()
2293 			ret = cpufreq_frequency_table_cpuinfo(policy,  in cpufreq_boost_set_sw()
2300 			policy->user_policy.max = policy->max;  in cpufreq_boost_set_sw()
2301 			__cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);  in cpufreq_boost_set_sw()