1 /*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 */
47 #include <linux/kernel.h>
48 #include <linux/module.h>
49 #include <linux/pci.h>
50 #include <linux/init.h>
51 #include <linux/types.h>
52 #include <linux/fs.h>
53 #include <linux/slab.h>
54 #include <linux/errno.h>
55 #include <linux/device.h>
56 #include <linux/dma-mapping.h>
57 #include <linux/platform_device.h>
58 #include <linux/workqueue.h>
59 #include <linux/io.h>
60 #include <adf_accel_devices.h>
61 #include <adf_common_drv.h>
62 #include <adf_cfg.h>
63 #include <adf_transport_access_macros.h>
64 #include "adf_dh895xcc_hw_data.h"
65 #include "adf_drv.h"
66
67 static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME;
68
69 #define ADF_SYSTEM_DEVICE(device_id) \
70 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
71
72 static const struct pci_device_id adf_pci_tbl[] = {
73 ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
74 {0,}
75 };
76 MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
77
78 static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent);
79 static void adf_remove(struct pci_dev *dev);
80
81 static struct pci_driver adf_driver = {
82 .id_table = adf_pci_tbl,
83 .name = adf_driver_name,
84 .probe = adf_probe,
85 .remove = adf_remove,
86 .sriov_configure = adf_sriov_configure,
87 };
88
adf_cleanup_pci_dev(struct adf_accel_dev * accel_dev)89 static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev)
90 {
91 pci_release_regions(accel_dev->accel_pci_dev.pci_dev);
92 pci_disable_device(accel_dev->accel_pci_dev.pci_dev);
93 }
94
adf_cleanup_accel(struct adf_accel_dev * accel_dev)95 static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
96 {
97 struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
98 int i;
99
100 for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
101 struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
102
103 if (bar->virt_addr)
104 pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr);
105 }
106
107 if (accel_dev->hw_device) {
108 switch (accel_pci_dev->pci_dev->device) {
109 case ADF_DH895XCC_PCI_DEVICE_ID:
110 adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
111 break;
112 default:
113 break;
114 }
115 kfree(accel_dev->hw_device);
116 accel_dev->hw_device = NULL;
117 }
118 adf_cfg_dev_remove(accel_dev);
119 debugfs_remove(accel_dev->debugfs_dir);
120 adf_devmgr_rm_dev(accel_dev, NULL);
121 }
122
adf_dev_configure(struct adf_accel_dev * accel_dev)123 static int adf_dev_configure(struct adf_accel_dev *accel_dev)
124 {
125 int cpus = num_online_cpus();
126 int banks = GET_MAX_BANKS(accel_dev);
127 int instances = min(cpus, banks);
128 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
129 int i;
130 unsigned long val;
131
132 if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC))
133 goto err;
134 if (adf_cfg_section_add(accel_dev, "Accelerator0"))
135 goto err;
136 for (i = 0; i < instances; i++) {
137 val = i;
138 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i);
139 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
140 key, (void *)&val, ADF_DEC))
141 goto err;
142
143 snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY,
144 i);
145 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
146 key, (void *)&val, ADF_DEC))
147 goto err;
148
149 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i);
150 val = 128;
151 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
152 key, (void *)&val, ADF_DEC))
153 goto err;
154
155 val = 512;
156 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i);
157 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
158 key, (void *)&val, ADF_DEC))
159 goto err;
160
161 val = 0;
162 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i);
163 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
164 key, (void *)&val, ADF_DEC))
165 goto err;
166
167 val = 2;
168 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i);
169 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
170 key, (void *)&val, ADF_DEC))
171 goto err;
172
173 val = 8;
174 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i);
175 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
176 key, (void *)&val, ADF_DEC))
177 goto err;
178
179 val = 10;
180 snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i);
181 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
182 key, (void *)&val, ADF_DEC))
183 goto err;
184
185 val = ADF_COALESCING_DEF_TIME;
186 snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i);
187 if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0",
188 key, (void *)&val, ADF_DEC))
189 goto err;
190 }
191
192 val = i;
193 if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC,
194 ADF_NUM_CY, (void *)&val, ADF_DEC))
195 goto err;
196
197 set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
198 return 0;
199 err:
200 dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
201 return -EINVAL;
202 }
203
adf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)204 static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
205 {
206 struct adf_accel_dev *accel_dev;
207 struct adf_accel_pci *accel_pci_dev;
208 struct adf_hw_device_data *hw_data;
209 char name[ADF_DEVICE_NAME_LENGTH];
210 unsigned int i, bar_nr;
211 int ret, bar_mask;
212
213 switch (ent->device) {
214 case ADF_DH895XCC_PCI_DEVICE_ID:
215 break;
216 default:
217 dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
218 return -ENODEV;
219 }
220
221 if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
222 /* If the accelerator is connected to a node with no memory
223 * there is no point in using the accelerator since the remote
224 * memory transaction will be very slow. */
225 dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
226 return -EINVAL;
227 }
228
229 accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL,
230 dev_to_node(&pdev->dev));
231 if (!accel_dev)
232 return -ENOMEM;
233
234 INIT_LIST_HEAD(&accel_dev->crypto_list);
235 accel_pci_dev = &accel_dev->accel_pci_dev;
236 accel_pci_dev->pci_dev = pdev;
237
238 /* Add accel device to accel table.
239 * This should be called before adf_cleanup_accel is called */
240 if (adf_devmgr_add_dev(accel_dev, NULL)) {
241 dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
242 kfree(accel_dev);
243 return -EFAULT;
244 }
245
246 accel_dev->owner = THIS_MODULE;
247 /* Allocate and configure device configuration structure */
248 hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL,
249 dev_to_node(&pdev->dev));
250 if (!hw_data) {
251 ret = -ENOMEM;
252 goto out_err;
253 }
254
255 accel_dev->hw_device = hw_data;
256 switch (ent->device) {
257 case ADF_DH895XCC_PCI_DEVICE_ID:
258 adf_init_hw_data_dh895xcc(accel_dev->hw_device);
259 break;
260 default:
261 return -ENODEV;
262 }
263 pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
264 pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET,
265 &hw_data->fuses);
266
267 /* Get Accelerators and Accelerators Engines masks */
268 hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses);
269 hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses);
270 accel_pci_dev->sku = hw_data->get_sku(hw_data);
271 /* If the device has no acceleration engines then ignore it. */
272 if (!hw_data->accel_mask || !hw_data->ae_mask ||
273 ((~hw_data->ae_mask) & 0x01)) {
274 dev_err(&pdev->dev, "No acceleration units found");
275 ret = -EFAULT;
276 goto out_err;
277 }
278
279 /* Create dev top level debugfs entry */
280 snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d",
281 ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
282 pdev->bus->number, PCI_SLOT(pdev->devfn),
283 PCI_FUNC(pdev->devfn));
284
285 accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
286 if (!accel_dev->debugfs_dir) {
287 dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name);
288 ret = -EINVAL;
289 goto out_err;
290 }
291
292 /* Create device configuration table */
293 ret = adf_cfg_dev_add(accel_dev);
294 if (ret)
295 goto out_err;
296
297 pcie_set_readrq(pdev, 1024);
298
299 /* enable PCI device */
300 if (pci_enable_device(pdev)) {
301 ret = -EFAULT;
302 goto out_err;
303 }
304
305 /* set dma identifier */
306 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
307 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) {
308 dev_err(&pdev->dev, "No usable DMA configuration\n");
309 ret = -EFAULT;
310 goto out_err_disable;
311 } else {
312 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
313 }
314
315 } else {
316 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
317 }
318
319 if (pci_request_regions(pdev, adf_driver_name)) {
320 ret = -EFAULT;
321 goto out_err_disable;
322 }
323
324 /* Read accelerator capabilities mask */
325 pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET,
326 &hw_data->accel_capabilities_mask);
327
328 /* Find and map all the device's BARS */
329 i = 0;
330 bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
331 for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask,
332 ADF_PCI_MAX_BARS * 2) {
333 struct adf_bar *bar = &accel_pci_dev->pci_bars[i++];
334
335 bar->base_addr = pci_resource_start(pdev, bar_nr);
336 if (!bar->base_addr)
337 break;
338 bar->size = pci_resource_len(pdev, bar_nr);
339 bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0);
340 if (!bar->virt_addr) {
341 dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr);
342 ret = -EFAULT;
343 goto out_err_free_reg;
344 }
345 }
346 pci_set_master(pdev);
347
348 if (adf_enable_aer(accel_dev, &adf_driver)) {
349 dev_err(&pdev->dev, "Failed to enable aer\n");
350 ret = -EFAULT;
351 goto out_err_free_reg;
352 }
353
354 if (pci_save_state(pdev)) {
355 dev_err(&pdev->dev, "Failed to save pci state\n");
356 ret = -ENOMEM;
357 goto out_err_free_reg;
358 }
359
360 ret = adf_dev_configure(accel_dev);
361 if (ret)
362 goto out_err_free_reg;
363
364 ret = adf_dev_init(accel_dev);
365 if (ret)
366 goto out_err_dev_shutdown;
367
368 ret = adf_dev_start(accel_dev);
369 if (ret)
370 goto out_err_dev_stop;
371
372 return ret;
373
374 out_err_dev_stop:
375 adf_dev_stop(accel_dev);
376 out_err_dev_shutdown:
377 adf_dev_shutdown(accel_dev);
378 out_err_free_reg:
379 pci_release_regions(accel_pci_dev->pci_dev);
380 out_err_disable:
381 pci_disable_device(accel_pci_dev->pci_dev);
382 out_err:
383 adf_cleanup_accel(accel_dev);
384 kfree(accel_dev);
385 return ret;
386 }
387
adf_remove(struct pci_dev * pdev)388 static void adf_remove(struct pci_dev *pdev)
389 {
390 struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
391
392 if (!accel_dev) {
393 pr_err("QAT: Driver removal failed\n");
394 return;
395 }
396 if (adf_dev_stop(accel_dev))
397 dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n");
398
399 adf_dev_shutdown(accel_dev);
400 adf_disable_aer(accel_dev);
401 adf_cleanup_accel(accel_dev);
402 adf_cleanup_pci_dev(accel_dev);
403 kfree(accel_dev);
404 }
405
adfdrv_init(void)406 static int __init adfdrv_init(void)
407 {
408 request_module("intel_qat");
409
410 if (pci_register_driver(&adf_driver)) {
411 pr_err("QAT: Driver initialization failed\n");
412 return -EFAULT;
413 }
414 return 0;
415 }
416
adfdrv_release(void)417 static void __exit adfdrv_release(void)
418 {
419 pci_unregister_driver(&adf_driver);
420 }
421
422 module_init(adfdrv_init);
423 module_exit(adfdrv_release);
424
425 MODULE_LICENSE("Dual BSD/GPL");
426 MODULE_AUTHOR("Intel");
427 MODULE_FIRMWARE(ADF_DH895XCC_FW);
428 MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
429 MODULE_VERSION(ADF_DRV_VERSION);
430