1#include <linux/seq_file.h>
2#include <linux/cpumask.h>
3#include <linux/kernel.h>
4#include <linux/string.h>
5#include <linux/errno.h>
6#include <linux/msi.h>
7#include <linux/irq.h>
8#include <linux/pci.h>
9
10#include <asm/hw_irq.h>
11#include <asm/irq_remapping.h>
12#include <asm/processor.h>
13#include <asm/x86_init.h>
14#include <asm/apic.h>
15#include <asm/hpet.h>
16
17#include "irq_remapping.h"
18
19int irq_remapping_enabled;
20int irq_remap_broken;
21int disable_sourceid_checking;
22int no_x2apic_optout;
23
24static int disable_irq_remap;
25static struct irq_remap_ops *remap_ops;
26
27static int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec);
28static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
29				  int index, int sub_handle);
30static int set_remapped_irq_affinity(struct irq_data *data,
31				     const struct cpumask *mask,
32				     bool force);
33
34static bool irq_remapped(struct irq_cfg *cfg)
35{
36	return (cfg->remapped == 1);
37}
38
39static void irq_remapping_disable_io_apic(void)
40{
41	/*
42	 * With interrupt-remapping, for now we will use virtual wire A
43	 * mode, as virtual wire B is little complex (need to configure
44	 * both IOAPIC RTE as well as interrupt-remapping table entry).
45	 * As this gets called during crash dump, keep this simple for
46	 * now.
47	 */
48	if (cpu_has_apic || apic_from_smp_config())
49		disconnect_bsp_APIC(0);
50}
51
52static int do_setup_msi_irqs(struct pci_dev *dev, int nvec)
53{
54	int ret, sub_handle, nvec_pow2, index = 0;
55	unsigned int irq;
56	struct msi_desc *msidesc;
57
58	msidesc = list_entry(dev->msi_list.next, struct msi_desc, list);
59
60	irq = irq_alloc_hwirqs(nvec, dev_to_node(&dev->dev));
61	if (irq == 0)
62		return -ENOSPC;
63
64	nvec_pow2 = __roundup_pow_of_two(nvec);
65	for (sub_handle = 0; sub_handle < nvec; sub_handle++) {
66		if (!sub_handle) {
67			index = msi_alloc_remapped_irq(dev, irq, nvec_pow2);
68			if (index < 0) {
69				ret = index;
70				goto error;
71			}
72		} else {
73			ret = msi_setup_remapped_irq(dev, irq + sub_handle,
74						     index, sub_handle);
75			if (ret < 0)
76				goto error;
77		}
78		ret = setup_msi_irq(dev, msidesc, irq, sub_handle);
79		if (ret < 0)
80			goto error;
81	}
82	return 0;
83
84error:
85	irq_free_hwirqs(irq, nvec);
86
87	/*
88	 * Restore altered MSI descriptor fields and prevent just destroyed
89	 * IRQs from tearing down again in default_teardown_msi_irqs()
90	 */
91	msidesc->irq = 0;
92
93	return ret;
94}
95
96static int do_setup_msix_irqs(struct pci_dev *dev, int nvec)
97{
98	int node, ret, sub_handle, index = 0;
99	struct msi_desc *msidesc;
100	unsigned int irq;
101
102	node		= dev_to_node(&dev->dev);
103	sub_handle	= 0;
104
105	list_for_each_entry(msidesc, &dev->msi_list, list) {
106
107		irq = irq_alloc_hwirq(node);
108		if (irq == 0)
109			return -1;
110
111		if (sub_handle == 0)
112			ret = index = msi_alloc_remapped_irq(dev, irq, nvec);
113		else
114			ret = msi_setup_remapped_irq(dev, irq, index, sub_handle);
115
116		if (ret < 0)
117			goto error;
118
119		ret = setup_msi_irq(dev, msidesc, irq, 0);
120		if (ret < 0)
121			goto error;
122
123		sub_handle += 1;
124		irq        += 1;
125	}
126
127	return 0;
128
129error:
130	irq_free_hwirq(irq);
131	return ret;
132}
133
134static int irq_remapping_setup_msi_irqs(struct pci_dev *dev,
135					int nvec, int type)
136{
137	if (type == PCI_CAP_ID_MSI)
138		return do_setup_msi_irqs(dev, nvec);
139	else
140		return do_setup_msix_irqs(dev, nvec);
141}
142
143static void eoi_ioapic_pin_remapped(int apic, int pin, int vector)
144{
145	/*
146	 * Intr-remapping uses pin number as the virtual vector
147	 * in the RTE. Actual vector is programmed in
148	 * intr-remapping table entry. Hence for the io-apic
149	 * EOI we use the pin number.
150	 */
151	io_apic_eoi(apic, pin);
152}
153
154static void __init irq_remapping_modify_x86_ops(void)
155{
156	x86_io_apic_ops.disable		= irq_remapping_disable_io_apic;
157	x86_io_apic_ops.set_affinity	= set_remapped_irq_affinity;
158	x86_io_apic_ops.setup_entry	= setup_ioapic_remapped_entry;
159	x86_io_apic_ops.eoi_ioapic_pin	= eoi_ioapic_pin_remapped;
160	x86_msi.setup_msi_irqs		= irq_remapping_setup_msi_irqs;
161	x86_msi.setup_hpet_msi		= setup_hpet_msi_remapped;
162	x86_msi.compose_msi_msg		= compose_remapped_msi_msg;
163}
164
165static __init int setup_nointremap(char *str)
166{
167	disable_irq_remap = 1;
168	return 0;
169}
170early_param("nointremap", setup_nointremap);
171
172static __init int setup_irqremap(char *str)
173{
174	if (!str)
175		return -EINVAL;
176
177	while (*str) {
178		if (!strncmp(str, "on", 2))
179			disable_irq_remap = 0;
180		else if (!strncmp(str, "off", 3))
181			disable_irq_remap = 1;
182		else if (!strncmp(str, "nosid", 5))
183			disable_sourceid_checking = 1;
184		else if (!strncmp(str, "no_x2apic_optout", 16))
185			no_x2apic_optout = 1;
186
187		str += strcspn(str, ",");
188		while (*str == ',')
189			str++;
190	}
191
192	return 0;
193}
194early_param("intremap", setup_irqremap);
195
196void set_irq_remapping_broken(void)
197{
198	irq_remap_broken = 1;
199}
200
201int __init irq_remapping_prepare(void)
202{
203	if (disable_irq_remap)
204		return -ENOSYS;
205
206	if (intel_irq_remap_ops.prepare() == 0)
207		remap_ops = &intel_irq_remap_ops;
208	else if (IS_ENABLED(CONFIG_AMD_IOMMU) &&
209		 amd_iommu_irq_ops.prepare() == 0)
210		remap_ops = &amd_iommu_irq_ops;
211	else
212		return -ENOSYS;
213
214	return 0;
215}
216
217int __init irq_remapping_enable(void)
218{
219	int ret;
220
221	if (!remap_ops->enable)
222		return -ENODEV;
223
224	ret = remap_ops->enable();
225
226	if (irq_remapping_enabled)
227		irq_remapping_modify_x86_ops();
228
229	return ret;
230}
231
232void irq_remapping_disable(void)
233{
234	if (irq_remapping_enabled && remap_ops->disable)
235		remap_ops->disable();
236}
237
238int irq_remapping_reenable(int mode)
239{
240	if (irq_remapping_enabled && remap_ops->reenable)
241		return remap_ops->reenable(mode);
242
243	return 0;
244}
245
246int __init irq_remap_enable_fault_handling(void)
247{
248	if (!irq_remapping_enabled)
249		return 0;
250
251	if (!remap_ops->enable_faulting)
252		return -ENODEV;
253
254	return remap_ops->enable_faulting();
255}
256
257int setup_ioapic_remapped_entry(int irq,
258				struct IO_APIC_route_entry *entry,
259				unsigned int destination, int vector,
260				struct io_apic_irq_attr *attr)
261{
262	if (!remap_ops->setup_ioapic_entry)
263		return -ENODEV;
264
265	return remap_ops->setup_ioapic_entry(irq, entry, destination,
266					     vector, attr);
267}
268
269static int set_remapped_irq_affinity(struct irq_data *data,
270				     const struct cpumask *mask, bool force)
271{
272	if (!config_enabled(CONFIG_SMP) || !remap_ops->set_affinity)
273		return 0;
274
275	return remap_ops->set_affinity(data, mask, force);
276}
277
278void free_remapped_irq(int irq)
279{
280	struct irq_cfg *cfg = irq_cfg(irq);
281
282	if (irq_remapped(cfg) && remap_ops->free_irq)
283		remap_ops->free_irq(irq);
284}
285
286void compose_remapped_msi_msg(struct pci_dev *pdev,
287			      unsigned int irq, unsigned int dest,
288			      struct msi_msg *msg, u8 hpet_id)
289{
290	struct irq_cfg *cfg = irq_cfg(irq);
291
292	if (!irq_remapped(cfg))
293		native_compose_msi_msg(pdev, irq, dest, msg, hpet_id);
294	else if (remap_ops->compose_msi_msg)
295		remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id);
296}
297
298static int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec)
299{
300	if (!remap_ops->msi_alloc_irq)
301		return -ENODEV;
302
303	return remap_ops->msi_alloc_irq(pdev, irq, nvec);
304}
305
306static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
307				  int index, int sub_handle)
308{
309	if (!remap_ops->msi_setup_irq)
310		return -ENODEV;
311
312	return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle);
313}
314
315int setup_hpet_msi_remapped(unsigned int irq, unsigned int id)
316{
317	int ret;
318
319	if (!remap_ops->alloc_hpet_msi)
320		return -ENODEV;
321
322	ret = remap_ops->alloc_hpet_msi(irq, id);
323	if (ret)
324		return -EINVAL;
325
326	return default_setup_hpet_msi(irq, id);
327}
328
329void panic_if_irq_remap(const char *msg)
330{
331	if (irq_remapping_enabled)
332		panic(msg);
333}
334
335static void ir_ack_apic_edge(struct irq_data *data)
336{
337	ack_APIC_irq();
338}
339
340static void ir_ack_apic_level(struct irq_data *data)
341{
342	ack_APIC_irq();
343	eoi_ioapic_irq(data->irq, irqd_cfg(data));
344}
345
346static void ir_print_prefix(struct irq_data *data, struct seq_file *p)
347{
348	seq_printf(p, " IR-%s", data->chip->name);
349}
350
351void irq_remap_modify_chip_defaults(struct irq_chip *chip)
352{
353	chip->irq_print_chip = ir_print_prefix;
354	chip->irq_ack = ir_ack_apic_edge;
355	chip->irq_eoi = ir_ack_apic_level;
356	chip->irq_set_affinity = x86_io_apic_ops.set_affinity;
357}
358
359bool setup_remapped_irq(int irq, struct irq_cfg *cfg, struct irq_chip *chip)
360{
361	if (!irq_remapped(cfg))
362		return false;
363	irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
364	irq_remap_modify_chip_defaults(chip);
365	return true;
366}
367