84 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu);
91 static void add_sgi_source(struct kvm_vcpu *vcpu, int irq, int source)  in add_sgi_source()  argument
93 	vcpu->kvm->arch.vgic.vm_ops.add_sgi_source(vcpu, irq, source);  in add_sgi_source()
96 static bool queue_sgi(struct kvm_vcpu *vcpu, int irq)  in queue_sgi()  argument
98 	return vcpu->kvm->arch.vgic.vm_ops.queue_sgi(vcpu, irq);  in queue_sgi()
170 				   int cpuid, int irq)  in vgic_bitmap_get_irq_val()  argument
172 	if (irq < VGIC_NR_PRIVATE_IRQS)  in vgic_bitmap_get_irq_val()
173 		return test_bit(irq, x->private + cpuid);  in vgic_bitmap_get_irq_val()
175 	return test_bit(irq - VGIC_NR_PRIVATE_IRQS, x->shared);  in vgic_bitmap_get_irq_val()
179 			     int irq, int val)  in vgic_bitmap_set_irq_val()  argument
183 	if (irq < VGIC_NR_PRIVATE_IRQS) {  in vgic_bitmap_set_irq_val()
187 		irq -= VGIC_NR_PRIVATE_IRQS;  in vgic_bitmap_set_irq_val()
191 		set_bit(irq, reg);  in vgic_bitmap_set_irq_val()
193 		clear_bit(irq, reg);  in vgic_bitmap_set_irq_val()
246 static bool vgic_irq_is_edge(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_is_edge()  argument
251 	irq_val = vgic_bitmap_get_irq_val(&dist->irq_cfg, vcpu->vcpu_id, irq);  in vgic_irq_is_edge()
255 static int vgic_irq_is_enabled(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_is_enabled()  argument
259 	return vgic_bitmap_get_irq_val(&dist->irq_enabled, vcpu->vcpu_id, irq);  in vgic_irq_is_enabled()
262 static int vgic_irq_is_queued(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_is_queued()  argument
266 	return vgic_bitmap_get_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq);  in vgic_irq_is_queued()
269 static int vgic_irq_is_active(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_is_active()  argument
273 	return vgic_bitmap_get_irq_val(&dist->irq_active, vcpu->vcpu_id, irq);  in vgic_irq_is_active()
276 static void vgic_irq_set_queued(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_set_queued()  argument
280 	vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 1);  in vgic_irq_set_queued()
283 static void vgic_irq_clear_queued(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_clear_queued()  argument
287 	vgic_bitmap_set_irq_val(&dist->irq_queued, vcpu->vcpu_id, irq, 0);  in vgic_irq_clear_queued()
290 static void vgic_irq_set_active(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_set_active()  argument
294 	vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 1);  in vgic_irq_set_active()
297 static void vgic_irq_clear_active(struct kvm_vcpu *vcpu, int irq)  in vgic_irq_clear_active()  argument
301 	vgic_bitmap_set_irq_val(&dist->irq_active, vcpu->vcpu_id, irq, 0);  in vgic_irq_clear_active()
304 static int vgic_dist_irq_get_level(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_get_level()  argument
308 	return vgic_bitmap_get_irq_val(&dist->irq_level, vcpu->vcpu_id, irq);  in vgic_dist_irq_get_level()
311 static void vgic_dist_irq_set_level(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_set_level()  argument
315 	vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 1);  in vgic_dist_irq_set_level()
318 static void vgic_dist_irq_clear_level(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_clear_level()  argument
322 	vgic_bitmap_set_irq_val(&dist->irq_level, vcpu->vcpu_id, irq, 0);  in vgic_dist_irq_clear_level()
325 static int vgic_dist_irq_soft_pend(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_soft_pend()  argument
329 	return vgic_bitmap_get_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq);  in vgic_dist_irq_soft_pend()
332 static void vgic_dist_irq_clear_soft_pend(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_clear_soft_pend()  argument
336 	vgic_bitmap_set_irq_val(&dist->irq_soft_pend, vcpu->vcpu_id, irq, 0);  in vgic_dist_irq_clear_soft_pend()
339 static int vgic_dist_irq_is_pending(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_is_pending()  argument
343 	return vgic_bitmap_get_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq);  in vgic_dist_irq_is_pending()
346 void vgic_dist_irq_set_pending(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_set_pending()  argument
350 	vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 1);  in vgic_dist_irq_set_pending()
353 void vgic_dist_irq_clear_pending(struct kvm_vcpu *vcpu, int irq)  in vgic_dist_irq_clear_pending()  argument
357 	vgic_bitmap_set_irq_val(&dist->irq_pending, vcpu->vcpu_id, irq, 0);  in vgic_dist_irq_clear_pending()
360 static void vgic_cpu_irq_set(struct kvm_vcpu *vcpu, int irq)  in vgic_cpu_irq_set()  argument
362 	if (irq < VGIC_NR_PRIVATE_IRQS)  in vgic_cpu_irq_set()
363 		set_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);  in vgic_cpu_irq_set()
365 		set_bit(irq - VGIC_NR_PRIVATE_IRQS,  in vgic_cpu_irq_set()
369 void vgic_cpu_irq_clear(struct kvm_vcpu *vcpu, int irq)  in vgic_cpu_irq_clear()  argument
371 	if (irq < VGIC_NR_PRIVATE_IRQS)  in vgic_cpu_irq_clear()
372 		clear_bit(irq, vcpu->arch.vgic_cpu.pending_percpu);  in vgic_cpu_irq_clear()
374 		clear_bit(irq - VGIC_NR_PRIVATE_IRQS,  in vgic_cpu_irq_clear()
378 static bool vgic_can_sample_irq(struct kvm_vcpu *vcpu, int irq)  in vgic_can_sample_irq()  argument
380 	return vgic_irq_is_edge(vcpu, irq) || !vgic_irq_is_queued(vcpu, irq);  in vgic_can_sample_irq()
678 		if (lr.irq < VGIC_NR_SGIS)  in vgic_unqueue_irqs()
679 			add_sgi_source(vcpu, lr.irq, lr.source);  in vgic_unqueue_irqs()
687 			vgic_irq_set_active(vcpu, lr.irq);  in vgic_unqueue_irqs()
698 			vgic_dist_irq_set_pending(vcpu, lr.irq);  in vgic_unqueue_irqs()
708 		vgic_retire_lr(i, lr.irq, vcpu);  in vgic_unqueue_irqs()
709 		vgic_irq_clear_queued(vcpu, lr.irq);  in vgic_unqueue_irqs()
734 	int irq;  in vgic_validate_access()  local
739 	irq = offset * 8 / range->bits_per_irq;  in vgic_validate_access()
740 	if (irq >= dist->nr_irqs)  in vgic_validate_access()
1067 static void vgic_retire_lr(int lr_nr, int irq, struct kvm_vcpu *vcpu)  in vgic_retire_lr()  argument
1075 	vgic_cpu->vgic_irq_lr_map[irq] = LR_EMPTY;  in vgic_retire_lr()
1096 		if (!vgic_irq_is_enabled(vcpu, vlr.irq)) {  in vgic_retire_disabled_irqs()
1097 			vgic_retire_lr(lr, vlr.irq, vcpu);  in vgic_retire_disabled_irqs()
1098 			if (vgic_irq_is_queued(vcpu, vlr.irq))  in vgic_retire_disabled_irqs()
1099 				vgic_irq_clear_queued(vcpu, vlr.irq);  in vgic_retire_disabled_irqs()
1104 static void vgic_queue_irq_to_lr(struct kvm_vcpu *vcpu, int irq,  in vgic_queue_irq_to_lr()  argument
1107 	if (vgic_irq_is_active(vcpu, irq)) {  in vgic_queue_irq_to_lr()
1110 		vgic_irq_clear_active(vcpu, irq);  in vgic_queue_irq_to_lr()
1112 	} else if (vgic_dist_irq_is_pending(vcpu, irq)) {  in vgic_queue_irq_to_lr()
1117 	if (!vgic_irq_is_edge(vcpu, irq))  in vgic_queue_irq_to_lr()
1129 bool vgic_queue_irq(struct kvm_vcpu *vcpu, u8 sgi_source_id, int irq)  in vgic_queue_irq()  argument
1138 	BUG_ON(sgi_source_id && irq >= VGIC_NR_SGIS);  in vgic_queue_irq()
1139 	BUG_ON(irq >= dist->nr_irqs);  in vgic_queue_irq()
1141 	kvm_debug("Queue IRQ%d\n", irq);  in vgic_queue_irq()
1143 	lr = vgic_cpu->vgic_irq_lr_map[irq];  in vgic_queue_irq()
1149 			kvm_debug("LR%d piggyback for IRQ%d\n", lr, vlr.irq);  in vgic_queue_irq()
1151 			vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);  in vgic_queue_irq()
1162 	kvm_debug("LR%d allocated for IRQ%d %x\n", lr, irq, sgi_source_id);  in vgic_queue_irq()
1163 	vgic_cpu->vgic_irq_lr_map[irq] = lr;  in vgic_queue_irq()
1166 	vlr.irq = irq;  in vgic_queue_irq()
1169 	vgic_queue_irq_to_lr(vcpu, irq, lr, vlr);  in vgic_queue_irq()
1174 static bool vgic_queue_hwirq(struct kvm_vcpu *vcpu, int irq)  in vgic_queue_hwirq()  argument
1176 	if (!vgic_can_sample_irq(vcpu, irq))  in vgic_queue_hwirq()
1179 	if (vgic_queue_irq(vcpu, 0, irq)) {  in vgic_queue_hwirq()
1180 		if (vgic_irq_is_edge(vcpu, irq)) {  in vgic_queue_hwirq()
1181 			vgic_dist_irq_clear_pending(vcpu, irq);  in vgic_queue_hwirq()
1182 			vgic_cpu_irq_clear(vcpu, irq);  in vgic_queue_hwirq()
1184 			vgic_irq_set_queued(vcpu, irq);  in vgic_queue_hwirq()
1279 			WARN_ON(vgic_irq_is_edge(vcpu, vlr.irq));  in vgic_process_maintenance()
1282 			vgic_irq_clear_queued(vcpu, vlr.irq);  in vgic_process_maintenance()
1298 			vgic_dist_irq_clear_soft_pend(vcpu, vlr.irq);  in vgic_process_maintenance()
1308 					     vlr.irq - VGIC_NR_PRIVATE_IRQS);  in vgic_process_maintenance()
1312 			if (vgic_dist_irq_get_level(vcpu, vlr.irq)) {  in vgic_process_maintenance()
1313 				vgic_cpu_irq_set(vcpu, vlr.irq);  in vgic_process_maintenance()
1316 				vgic_dist_irq_clear_pending(vcpu, vlr.irq);  in vgic_process_maintenance()
1317 				vgic_cpu_irq_clear(vcpu, vlr.irq);  in vgic_process_maintenance()
1367 		BUG_ON(vlr.irq >= dist->nr_irqs);  in __kvm_vgic_sync_hwstate()
1368 		vgic_cpu->vgic_irq_lr_map[vlr.irq] = LR_EMPTY;  in __kvm_vgic_sync_hwstate()
1433 static int vgic_validate_injection(struct kvm_vcpu *vcpu, int irq, int level)  in vgic_validate_injection()  argument
1435 	int edge_triggered = vgic_irq_is_edge(vcpu, irq);  in vgic_validate_injection()
1443 		int state = vgic_dist_irq_is_pending(vcpu, irq);  in vgic_validate_injection()
1446 		int state = vgic_dist_irq_get_level(vcpu, irq);  in vgic_validate_injection()
1577 static irqreturn_t vgic_maintenance_handler(int irq, void *data)  in vgic_maintenance_handler()  argument
2156 		u32 irq, int level, bool line_status)  in kvm_set_irq()  argument
2158 	unsigned int spi = irq + VGIC_NR_PRIVATE_IRQS;  in kvm_set_irq()
2160 	trace_kvm_set_irq(irq, level, irq_source_id);  in kvm_set_irq()