1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3 * Copyright(c) 2018 Intel Corporation.
4 *
5 * This file is provided under a dual BSD/GPLv2 license. When using or
6 * redistributing this file, you may do so under either license.
7 *
8 * GPL LICENSE SUMMARY
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
18 *
19 * BSD LICENSE
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 *
25 * - Redistributions of source code must retain the above copyright
26 * notice, this list of conditions and the following disclaimer.
27 * - Redistributions in binary form must reproduce the above copyright
28 * notice, this list of conditions and the following disclaimer in
29 * the documentation and/or other materials provided with the
30 * distribution.
31 * - Neither the name of Intel Corporation nor the names of its
32 * contributors may be used to endorse or promote products derived
33 * from this software without specific prior written permission.
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 *
47 */
48
49 #include "hfi.h"
50 #include "affinity.h"
51 #include "sdma.h"
52
53 /**
54 * msix_initialize() - Calculate, request and configure MSIx IRQs
55 * @dd: valid hfi1 devdata
56 *
57 */
58 int msix_initialize(struct hfi1_devdata *dd)
59 {
60 u32 total;
61 int ret;
62 struct hfi1_msix_entry *entries;
63
64 /*
65 * MSIx interrupt count:
66 * one for the general, "slow path" interrupt
67 * one per used SDMA engine
68 * one per kernel receive context
69 * one for each VNIC context
70 * ...any new IRQs should be added here.
71 */
72 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_vnic_contexts;
73
74 if (total >= CCE_NUM_MSIX_VECTORS)
75 return -EINVAL;
76
77 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX);
78 if (ret < 0) {
79 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret);
80 return ret;
81 }
82
83 entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries),
84 GFP_KERNEL);
85 if (!entries) {
86 pci_free_irq_vectors(dd->pcidev);
87 return -ENOMEM;
88 }
89
90 dd->msix_info.msix_entries = entries;
91 spin_lock_init(&dd->msix_info.msix_lock);
92 bitmap_zero(dd->msix_info.in_use_msix, total);
93 dd->msix_info.max_requested = total;
94 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
95
96 return 0;
97 }
98
99 /**
100 * msix_request_irq() - Allocate a free MSIx IRQ
101 * @dd: valid devdata
102 * @arg: context information for the IRQ
103 * @handler: IRQ handler
104 * @thread: IRQ thread handler (could be NULL)
105 * @idx: zero base idx if multiple devices are needed
106 * @type: affinty IRQ type
107 *
108 * Allocated an MSIx vector if available, and then create the appropriate
109 * meta data needed to keep track of the pci IRQ request.
110 *
111 * Return:
112 * < 0 Error
113 * >= 0 MSIx vector
114 *
115 */
116 static int msix_request_irq(struct hfi1_devdata *dd, void *arg,
117 irq_handler_t handler, irq_handler_t thread,
118 u32 idx, enum irq_type type)
119 {
120 unsigned long nr;
121 int irq;
122 int ret;
123 const char *err_info;
124 char name[MAX_NAME_SIZE];
125 struct hfi1_msix_entry *me;
126
127 /* Allocate an MSIx vector */
128 spin_lock(&dd->msix_info.msix_lock);
129 nr = find_first_zero_bit(dd->msix_info.in_use_msix,
130 dd->msix_info.max_requested);
131 if (nr < dd->msix_info.max_requested)
132 __set_bit(nr, dd->msix_info.in_use_msix);
133 spin_unlock(&dd->msix_info.msix_lock);
134
135 if (nr == dd->msix_info.max_requested)
136 return -ENOSPC;
137
138 /* Specific verification and determine the name */
139 switch (type) {
140 case IRQ_GENERAL:
141 /* general interrupt must be MSIx vector 0 */
142 if (nr) {
143 spin_lock(&dd->msix_info.msix_lock);
144 __clear_bit(nr, dd->msix_info.in_use_msix);
145 spin_unlock(&dd->msix_info.msix_lock);
146 dd_dev_err(dd, "Invalid index %lu for GENERAL IRQ\n",
147 nr);
148 return -EINVAL;
149 }
150 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit);
151 err_info = "general";
152 break;
153 case IRQ_SDMA:
154 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d",
155 dd->unit, idx);
156 err_info = "sdma";
157 break;
158 case IRQ_RCVCTXT:
159 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d",
160 dd->unit, idx);
161 err_info = "receive context";
162 break;
163 case IRQ_OTHER:
164 default:
165 return -EINVAL;
166 }
167 name[sizeof(name) - 1] = 0;
168
169 irq = pci_irq_vector(dd->pcidev, nr);
170 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name);
171 if (ret) {
172 dd_dev_err(dd,
173 "%s: request for IRQ %d failed, MSIx %d, err %d\n",
174 err_info, irq, idx, ret);
175 spin_lock(&dd->msix_info.msix_lock);
176 __clear_bit(nr, dd->msix_info.in_use_msix);
177 spin_unlock(&dd->msix_info.msix_lock);
178 return ret;
179 }
180
181 /*
182 * assign arg after pci_request_irq call, so it will be
183 * cleaned up
184 */
185 me = &dd->msix_info.msix_entries[nr];
186 me->irq = irq;
187 me->arg = arg;
188 me->type = type;
189
190 /* This is a request, so a failure is not fatal */
191 ret = hfi1_get_irq_affinity(dd, me);
192 if (ret)
193 dd_dev_err(dd, "unable to pin IRQ %d\n", ret);
194
195 return nr;
196 }
197
198 /**
199 * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs
200 * @rcd: valid rcd context
201 *
202 */
203 int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)
204 {
205 int nr;
206
207 nr = msix_request_irq(rcd->dd, rcd, receive_context_interrupt,
208 receive_context_thread, rcd->ctxt, IRQ_RCVCTXT);
209 if (nr < 0)
210 return nr;
211
212 /*
213 * Set the interrupt register and mask for this
214 * context's interrupt.
215 */
216 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64;
217 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64);
218 rcd->msix_intr = nr;
219 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr);
220
221 return 0;
222 }
223
224 /**
225 * msix_request_smda_ira() - Helper for getting SDMA IRQ resources
226 * @sde: valid sdma engine
227 *
228 */
229 int msix_request_sdma_irq(struct sdma_engine *sde)
230 {
231 int nr;
232
233 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL,
234 sde->this_idx, IRQ_SDMA);
235 if (nr < 0)
236 return nr;
237 sde->msix_intr = nr;
238 remap_sdma_interrupts(sde->dd, sde->this_idx, nr);
239
240 return 0;
241 }
242
243 /**
244 * enable_sdma_src() - Helper to enable SDMA IRQ srcs
245 * @dd: valid devdata structure
246 * @i: index of SDMA engine
247 */
248 static void enable_sdma_srcs(struct hfi1_devdata *dd, int i)
249 {
250 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true);
251 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i,
252 IS_SDMA_PROGRESS_START + i, true);
253 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true);
254 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i,
255 true);
256 }
257
258 /**
259 * msix_request_irqs() - Allocate all MSIx IRQs
260 * @dd: valid devdata structure
261 *
262 * Helper function to request the used MSIx IRQs.
263 *
264 */
265 int msix_request_irqs(struct hfi1_devdata *dd)
266 {
267 int i;
268 int ret;
269
270 ret = msix_request_irq(dd, dd, general_interrupt, NULL, 0, IRQ_GENERAL);
271 if (ret < 0)
272 return ret;
273
274 for (i = 0; i < dd->num_sdma; i++) {
275 struct sdma_engine *sde = &dd->per_sdma[i];
276
277 ret = msix_request_sdma_irq(sde);
278 if (ret)
279 return ret;
280 enable_sdma_srcs(sde->dd, i);
281 }
282
283 for (i = 0; i < dd->n_krcv_queues; i++) {
284 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i);
285
286 if (rcd)
287 ret = msix_request_rcd_irq(rcd);
288 hfi1_rcd_put(rcd);
289 if (ret)
290 return ret;
291 }
292
293 return 0;
294 }
295
296 /**
297 * msix_free_irq() - Free the specified MSIx resources and IRQ
298 * @dd: valid devdata
299 * @msix_intr: MSIx vector to free.
300 *
301 */
302 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr)
303 {
304 struct hfi1_msix_entry *me;
305
306 if (msix_intr >= dd->msix_info.max_requested)
307 return;
308
309 me = &dd->msix_info.msix_entries[msix_intr];
310
311 if (!me->arg) /* => no irq, no affinity */
312 return;
313
314 hfi1_put_irq_affinity(dd, me);
315 pci_free_irq(dd->pcidev, msix_intr, me->arg);
316
317 me->arg = NULL;
318
319 spin_lock(&dd->msix_info.msix_lock);
320 __clear_bit(msix_intr, dd->msix_info.in_use_msix);
321 spin_unlock(&dd->msix_info.msix_lock);
322 }
323
324 /**
325 * hfi1_clean_up_msix_interrupts() - Free all MSIx IRQ resources
326 * @dd: valid device data data structure
327 *
328 * Free the MSIx and associated PCI resources, if they have been allocated.
329 */
330 void msix_clean_up_interrupts(struct hfi1_devdata *dd)
331 {
332 int i;
333 struct hfi1_msix_entry *me = dd->msix_info.msix_entries;
334
335 /* remove irqs - must happen before disabling/turning off */
336 for (i = 0; i < dd->msix_info.max_requested; i++, me++)
337 msix_free_irq(dd, i);
338
339 /* clean structures */
340 kfree(dd->msix_info.msix_entries);
341 dd->msix_info.msix_entries = NULL;
342 dd->msix_info.max_requested = 0;
343
344 pci_free_irq_vectors(dd->pcidev);
345 }
346
347 /**
348 * msix_vnic_syncrhonize_irq() - Vnic IRQ synchronize
349 * @dd: valid devdata
350 */
351 void msix_vnic_synchronize_irq(struct hfi1_devdata *dd)
352 {
353 int i;
354
355 for (i = 0; i < dd->vnic.num_ctxt; i++) {
356 struct hfi1_ctxtdata *rcd = dd->vnic.ctxt[i];
357 struct hfi1_msix_entry *me;
358
359 me = &dd->msix_info.msix_entries[rcd->msix_intr];
360
361 synchronize_irq(me->irq);
362 }
363 }