32 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);  in irq_set_chip()  local
34 	if (!desc)  in irq_set_chip()
40 	desc->irq_data.chip = chip;  in irq_set_chip()
41 	irq_put_desc_unlock(desc, flags);  in irq_set_chip()
59 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);  in irq_set_irq_type()  local
62 	if (!desc)  in irq_set_irq_type()
66 	ret = __irq_set_trigger(desc, irq, type);  in irq_set_irq_type()
67 	irq_put_desc_busunlock(desc, flags);  in irq_set_irq_type()
82 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);  in irq_set_handler_data()  local
84 	if (!desc)  in irq_set_handler_data()
86 	desc->irq_data.handler_data = data;  in irq_set_handler_data()
87 	irq_put_desc_unlock(desc, flags);  in irq_set_handler_data()
104 …struct irq_desc *desc = irq_get_desc_lock(irq_base + irq_offset, &flags, IRQ_GET_DESC_CHECK_GLOBAL…  in irq_set_msi_desc_off()  local
106 	if (!desc)  in irq_set_msi_desc_off()
108 	desc->irq_data.msi_desc = entry;  in irq_set_msi_desc_off()
111 	irq_put_desc_unlock(desc, flags);  in irq_set_msi_desc_off()
137 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);  in irq_set_chip_data()  local
139 	if (!desc)  in irq_set_chip_data()
141 	desc->irq_data.chip_data = data;  in irq_set_chip_data()
142 	irq_put_desc_unlock(desc, flags);  in irq_set_chip_data()
149 	struct irq_desc *desc = irq_to_desc(irq);  in irq_get_irq_data()  local
151 	return desc ? &desc->irq_data : NULL;  in irq_get_irq_data()
155 static void irq_state_clr_disabled(struct irq_desc *desc)  in irq_state_clr_disabled()  argument
157 	irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);  in irq_state_clr_disabled()
160 static void irq_state_set_disabled(struct irq_desc *desc)  in irq_state_set_disabled()  argument
162 	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);  in irq_state_set_disabled()
165 static void irq_state_clr_masked(struct irq_desc *desc)  in irq_state_clr_masked()  argument
167 	irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);  in irq_state_clr_masked()
170 static void irq_state_set_masked(struct irq_desc *desc)  in irq_state_set_masked()  argument
172 	irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);  in irq_state_set_masked()
175 int irq_startup(struct irq_desc *desc, bool resend)  in irq_startup()  argument
179 	irq_state_clr_disabled(desc);  in irq_startup()
180 	desc->depth = 0;  in irq_startup()
182 	irq_domain_activate_irq(&desc->irq_data);  in irq_startup()
183 	if (desc->irq_data.chip->irq_startup) {  in irq_startup()
184 		ret = desc->irq_data.chip->irq_startup(&desc->irq_data);  in irq_startup()
185 		irq_state_clr_masked(desc);  in irq_startup()
187 		irq_enable(desc);  in irq_startup()
190 		check_irq_resend(desc, desc->irq_data.irq);  in irq_startup()
194 void irq_shutdown(struct irq_desc *desc)  in irq_shutdown()  argument
196 	irq_state_set_disabled(desc);  in irq_shutdown()
197 	desc->depth = 1;  in irq_shutdown()
198 	if (desc->irq_data.chip->irq_shutdown)  in irq_shutdown()
199 		desc->irq_data.chip->irq_shutdown(&desc->irq_data);  in irq_shutdown()
200 	else if (desc->irq_data.chip->irq_disable)  in irq_shutdown()
201 		desc->irq_data.chip->irq_disable(&desc->irq_data);  in irq_shutdown()
203 		desc->irq_data.chip->irq_mask(&desc->irq_data);  in irq_shutdown()
204 	irq_domain_deactivate_irq(&desc->irq_data);  in irq_shutdown()
205 	irq_state_set_masked(desc);  in irq_shutdown()
208 void irq_enable(struct irq_desc *desc)  in irq_enable()  argument
210 	irq_state_clr_disabled(desc);  in irq_enable()
211 	if (desc->irq_data.chip->irq_enable)  in irq_enable()
212 		desc->irq_data.chip->irq_enable(&desc->irq_data);  in irq_enable()
214 		desc->irq_data.chip->irq_unmask(&desc->irq_data);  in irq_enable()
215 	irq_state_clr_masked(desc);  in irq_enable()
231 void irq_disable(struct irq_desc *desc)  in irq_disable()  argument
233 	irq_state_set_disabled(desc);  in irq_disable()
234 	if (desc->irq_data.chip->irq_disable) {  in irq_disable()
235 		desc->irq_data.chip->irq_disable(&desc->irq_data);  in irq_disable()
236 		irq_state_set_masked(desc);  in irq_disable()
240 void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)  in irq_percpu_enable()  argument
242 	if (desc->irq_data.chip->irq_enable)  in irq_percpu_enable()
243 		desc->irq_data.chip->irq_enable(&desc->irq_data);  in irq_percpu_enable()
245 		desc->irq_data.chip->irq_unmask(&desc->irq_data);  in irq_percpu_enable()
246 	cpumask_set_cpu(cpu, desc->percpu_enabled);  in irq_percpu_enable()
249 void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)  in irq_percpu_disable()  argument
251 	if (desc->irq_data.chip->irq_disable)  in irq_percpu_disable()
252 		desc->irq_data.chip->irq_disable(&desc->irq_data);  in irq_percpu_disable()
254 		desc->irq_data.chip->irq_mask(&desc->irq_data);  in irq_percpu_disable()
255 	cpumask_clear_cpu(cpu, desc->percpu_enabled);  in irq_percpu_disable()
258 static inline void mask_ack_irq(struct irq_desc *desc)  in mask_ack_irq()  argument
260 	if (desc->irq_data.chip->irq_mask_ack)  in mask_ack_irq()
261 		desc->irq_data.chip->irq_mask_ack(&desc->irq_data);  in mask_ack_irq()
263 		desc->irq_data.chip->irq_mask(&desc->irq_data);  in mask_ack_irq()
264 		if (desc->irq_data.chip->irq_ack)  in mask_ack_irq()
265 			desc->irq_data.chip->irq_ack(&desc->irq_data);  in mask_ack_irq()
267 	irq_state_set_masked(desc);  in mask_ack_irq()
270 void mask_irq(struct irq_desc *desc)  in mask_irq()  argument
272 	if (desc->irq_data.chip->irq_mask) {  in mask_irq()
273 		desc->irq_data.chip->irq_mask(&desc->irq_data);  in mask_irq()
274 		irq_state_set_masked(desc);  in mask_irq()
278 void unmask_irq(struct irq_desc *desc)  in unmask_irq()  argument
280 	if (desc->irq_data.chip->irq_unmask) {  in unmask_irq()
281 		desc->irq_data.chip->irq_unmask(&desc->irq_data);  in unmask_irq()
282 		irq_state_clr_masked(desc);  in unmask_irq()
286 void unmask_threaded_irq(struct irq_desc *desc)  in unmask_threaded_irq()  argument
288 	struct irq_chip *chip = desc->irq_data.chip;  in unmask_threaded_irq()
291 		chip->irq_eoi(&desc->irq_data);  in unmask_threaded_irq()
294 		chip->irq_unmask(&desc->irq_data);  in unmask_threaded_irq()
295 		irq_state_clr_masked(desc);  in unmask_threaded_irq()
309 	struct irq_desc *desc = irq_to_desc(irq);  in handle_nested_irq()  local
315 	raw_spin_lock_irq(&desc->lock);  in handle_nested_irq()
317 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);  in handle_nested_irq()
318 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_nested_irq()
320 	action = desc->action;  in handle_nested_irq()
321 	if (unlikely(!action || irqd_irq_disabled(&desc->irq_data))) {  in handle_nested_irq()
322 		desc->istate |= IRQS_PENDING;  in handle_nested_irq()
326 	irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);  in handle_nested_irq()
327 	raw_spin_unlock_irq(&desc->lock);  in handle_nested_irq()
331 		note_interrupt(irq, desc, action_ret);  in handle_nested_irq()
333 	raw_spin_lock_irq(&desc->lock);  in handle_nested_irq()
334 	irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);  in handle_nested_irq()
337 	raw_spin_unlock_irq(&desc->lock);  in handle_nested_irq()
341 static bool irq_check_poll(struct irq_desc *desc)  in irq_check_poll()  argument
343 	if (!(desc->istate & IRQS_POLL_INPROGRESS))  in irq_check_poll()
345 	return irq_wait_for_poll(desc);  in irq_check_poll()
348 static bool irq_may_run(struct irq_desc *desc)  in irq_may_run()  argument
356 	if (!irqd_has_set(&desc->irq_data, mask))  in irq_may_run()
364 	if (irq_pm_check_wakeup(desc))  in irq_may_run()
370 	return irq_check_poll(desc);  in irq_may_run()
386 handle_simple_irq(unsigned int irq, struct irq_desc *desc)  in handle_simple_irq()  argument
388 	raw_spin_lock(&desc->lock);  in handle_simple_irq()
390 	if (!irq_may_run(desc))  in handle_simple_irq()
393 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);  in handle_simple_irq()
394 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_simple_irq()
396 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {  in handle_simple_irq()
397 		desc->istate |= IRQS_PENDING;  in handle_simple_irq()
401 	handle_irq_event(desc);  in handle_simple_irq()
404 	raw_spin_unlock(&desc->lock);  in handle_simple_irq()
412 static void cond_unmask_irq(struct irq_desc *desc)  in cond_unmask_irq()  argument
421 	if (!irqd_irq_disabled(&desc->irq_data) &&  in cond_unmask_irq()
422 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)  in cond_unmask_irq()
423 		unmask_irq(desc);  in cond_unmask_irq()
437 handle_level_irq(unsigned int irq, struct irq_desc *desc)  in handle_level_irq()  argument
439 	raw_spin_lock(&desc->lock);  in handle_level_irq()
440 	mask_ack_irq(desc);  in handle_level_irq()
442 	if (!irq_may_run(desc))  in handle_level_irq()
445 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);  in handle_level_irq()
446 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_level_irq()
452 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {  in handle_level_irq()
453 		desc->istate |= IRQS_PENDING;  in handle_level_irq()
457 	handle_irq_event(desc);  in handle_level_irq()
459 	cond_unmask_irq(desc);  in handle_level_irq()
462 	raw_spin_unlock(&desc->lock);  in handle_level_irq()
467 static inline void preflow_handler(struct irq_desc *desc)  in preflow_handler()  argument
469 	if (desc->preflow_handler)  in preflow_handler()
470 		desc->preflow_handler(&desc->irq_data);  in preflow_handler()
473 static inline void preflow_handler(struct irq_desc *desc) { }  in preflow_handler()  argument
476 static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)  in cond_unmask_eoi_irq()  argument
478 	if (!(desc->istate & IRQS_ONESHOT)) {  in cond_unmask_eoi_irq()
479 		chip->irq_eoi(&desc->irq_data);  in cond_unmask_eoi_irq()
488 	if (!irqd_irq_disabled(&desc->irq_data) &&  in cond_unmask_eoi_irq()
489 	    irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {  in cond_unmask_eoi_irq()
490 		chip->irq_eoi(&desc->irq_data);  in cond_unmask_eoi_irq()
491 		unmask_irq(desc);  in cond_unmask_eoi_irq()
493 		chip->irq_eoi(&desc->irq_data);  in cond_unmask_eoi_irq()
508 handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)  in handle_fasteoi_irq()  argument
510 	struct irq_chip *chip = desc->irq_data.chip;  in handle_fasteoi_irq()
512 	raw_spin_lock(&desc->lock);  in handle_fasteoi_irq()
514 	if (!irq_may_run(desc))  in handle_fasteoi_irq()
517 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);  in handle_fasteoi_irq()
518 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_fasteoi_irq()
524 	if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {  in handle_fasteoi_irq()
525 		desc->istate |= IRQS_PENDING;  in handle_fasteoi_irq()
526 		mask_irq(desc);  in handle_fasteoi_irq()
530 	if (desc->istate & IRQS_ONESHOT)  in handle_fasteoi_irq()
531 		mask_irq(desc);  in handle_fasteoi_irq()
533 	preflow_handler(desc);  in handle_fasteoi_irq()
534 	handle_irq_event(desc);  in handle_fasteoi_irq()
536 	cond_unmask_eoi_irq(desc, chip);  in handle_fasteoi_irq()
538 	raw_spin_unlock(&desc->lock);  in handle_fasteoi_irq()
542 		chip->irq_eoi(&desc->irq_data);  in handle_fasteoi_irq()
543 	raw_spin_unlock(&desc->lock);  in handle_fasteoi_irq()
564 handle_edge_irq(unsigned int irq, struct irq_desc *desc)  in handle_edge_irq()  argument
566 	raw_spin_lock(&desc->lock);  in handle_edge_irq()
568 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);  in handle_edge_irq()
570 	if (!irq_may_run(desc)) {  in handle_edge_irq()
571 		desc->istate |= IRQS_PENDING;  in handle_edge_irq()
572 		mask_ack_irq(desc);  in handle_edge_irq()
580 	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {  in handle_edge_irq()
581 		desc->istate |= IRQS_PENDING;  in handle_edge_irq()
582 		mask_ack_irq(desc);  in handle_edge_irq()
586 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_edge_irq()
589 	desc->irq_data.chip->irq_ack(&desc->irq_data);  in handle_edge_irq()
592 		if (unlikely(!desc->action)) {  in handle_edge_irq()
593 			mask_irq(desc);  in handle_edge_irq()
602 		if (unlikely(desc->istate & IRQS_PENDING)) {  in handle_edge_irq()
603 			if (!irqd_irq_disabled(&desc->irq_data) &&  in handle_edge_irq()
604 			    irqd_irq_masked(&desc->irq_data))  in handle_edge_irq()
605 				unmask_irq(desc);  in handle_edge_irq()
608 		handle_irq_event(desc);  in handle_edge_irq()
610 	} while ((desc->istate & IRQS_PENDING) &&  in handle_edge_irq()
611 		 !irqd_irq_disabled(&desc->irq_data));  in handle_edge_irq()
614 	raw_spin_unlock(&desc->lock);  in handle_edge_irq()
627 void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc)  in handle_edge_eoi_irq()  argument
629 	struct irq_chip *chip = irq_desc_get_chip(desc);  in handle_edge_eoi_irq()
631 	raw_spin_lock(&desc->lock);  in handle_edge_eoi_irq()
633 	desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);  in handle_edge_eoi_irq()
635 	if (!irq_may_run(desc)) {  in handle_edge_eoi_irq()
636 		desc->istate |= IRQS_PENDING;  in handle_edge_eoi_irq()
644 	if (irqd_irq_disabled(&desc->irq_data) || !desc->action) {  in handle_edge_eoi_irq()
645 		desc->istate |= IRQS_PENDING;  in handle_edge_eoi_irq()
649 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_edge_eoi_irq()
652 		if (unlikely(!desc->action))  in handle_edge_eoi_irq()
655 		handle_irq_event(desc);  in handle_edge_eoi_irq()
657 	} while ((desc->istate & IRQS_PENDING) &&  in handle_edge_eoi_irq()
658 		 !irqd_irq_disabled(&desc->irq_data));  in handle_edge_eoi_irq()
661 	chip->irq_eoi(&desc->irq_data);  in handle_edge_eoi_irq()
662 	raw_spin_unlock(&desc->lock);  in handle_edge_eoi_irq()
674 handle_percpu_irq(unsigned int irq, struct irq_desc *desc)  in handle_percpu_irq()  argument
676 	struct irq_chip *chip = irq_desc_get_chip(desc);  in handle_percpu_irq()
678 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_percpu_irq()
681 		chip->irq_ack(&desc->irq_data);  in handle_percpu_irq()
683 	handle_irq_event_percpu(desc, desc->action);  in handle_percpu_irq()
686 		chip->irq_eoi(&desc->irq_data);  in handle_percpu_irq()
701 void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc)  in handle_percpu_devid_irq()  argument
703 	struct irq_chip *chip = irq_desc_get_chip(desc);  in handle_percpu_devid_irq()
704 	struct irqaction *action = desc->action;  in handle_percpu_devid_irq()
708 	kstat_incr_irqs_this_cpu(irq, desc);  in handle_percpu_devid_irq()
711 		chip->irq_ack(&desc->irq_data);  in handle_percpu_devid_irq()
718 		chip->irq_eoi(&desc->irq_data);  in handle_percpu_devid_irq()
726 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, 0);  in __irq_set_handler()  local
728 	if (!desc)  in __irq_set_handler()
734 		struct irq_data *irq_data = &desc->irq_data;  in __irq_set_handler()
763 		if (desc->irq_data.chip != &no_irq_chip)  in __irq_set_handler()
764 			mask_ack_irq(desc);  in __irq_set_handler()
765 		irq_state_set_disabled(desc);  in __irq_set_handler()
766 		desc->depth = 1;  in __irq_set_handler()
768 	desc->handle_irq = handle;  in __irq_set_handler()
769 	desc->name = name;  in __irq_set_handler()
772 		irq_settings_set_noprobe(desc);  in __irq_set_handler()
773 		irq_settings_set_norequest(desc);  in __irq_set_handler()
774 		irq_settings_set_nothread(desc);  in __irq_set_handler()
775 		irq_startup(desc, true);  in __irq_set_handler()
778 	irq_put_desc_busunlock(desc, flags);  in __irq_set_handler()
794 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);  in irq_modify_status()  local
796 	if (!desc)  in irq_modify_status()
798 	irq_settings_clr_and_set(desc, clr, set);  in irq_modify_status()
800 	irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |  in irq_modify_status()
802 	if (irq_settings_has_no_balance_set(desc))  in irq_modify_status()
803 		irqd_set(&desc->irq_data, IRQD_NO_BALANCING);  in irq_modify_status()
804 	if (irq_settings_is_per_cpu(desc))  in irq_modify_status()
805 		irqd_set(&desc->irq_data, IRQD_PER_CPU);  in irq_modify_status()
806 	if (irq_settings_can_move_pcntxt(desc))  in irq_modify_status()
807 		irqd_set(&desc->irq_data, IRQD_MOVE_PCNTXT);  in irq_modify_status()
808 	if (irq_settings_is_level(desc))  in irq_modify_status()
809 		irqd_set(&desc->irq_data, IRQD_LEVEL);  in irq_modify_status()
811 	irqd_set(&desc->irq_data, irq_settings_get_trigger_mask(desc));  in irq_modify_status()
813 	irq_put_desc_unlock(desc, flags);  in irq_modify_status()
825 	struct irq_desc *desc;  in irq_cpu_online()  local
831 		desc = irq_to_desc(irq);  in irq_cpu_online()
832 		if (!desc)  in irq_cpu_online()
835 		raw_spin_lock_irqsave(&desc->lock, flags);  in irq_cpu_online()
837 		chip = irq_data_get_irq_chip(&desc->irq_data);  in irq_cpu_online()
840 		     !irqd_irq_disabled(&desc->irq_data)))  in irq_cpu_online()
841 			chip->irq_cpu_online(&desc->irq_data);  in irq_cpu_online()
843 		raw_spin_unlock_irqrestore(&desc->lock, flags);  in irq_cpu_online()
855 	struct irq_desc *desc;  in irq_cpu_offline()  local
861 		desc = irq_to_desc(irq);  in irq_cpu_offline()
862 		if (!desc)  in irq_cpu_offline()
865 		raw_spin_lock_irqsave(&desc->lock, flags);  in irq_cpu_offline()
867 		chip = irq_data_get_irq_chip(&desc->irq_data);  in irq_cpu_offline()
870 		     !irqd_irq_disabled(&desc->irq_data)))  in irq_cpu_offline()
871 			chip->irq_cpu_offline(&desc->irq_data);  in irq_cpu_offline()
873 		raw_spin_unlock_irqrestore(&desc->lock, flags);  in irq_cpu_offline()