32 #define CXL_READ_VSEC_LENGTH(dev, vsec, dest)			\  argument
34 		pci_read_config_word(dev, vsec + 0x6, dest);	\
37 #define CXL_READ_VSEC_NAFUS(dev, vsec, dest) \  argument
38 	pci_read_config_byte(dev, vsec + 0x8, dest)
40 #define CXL_READ_VSEC_STATUS(dev, vsec, dest) \  argument
41 	pci_read_config_byte(dev, vsec + 0x9, dest)
53 #define CXL_READ_VSEC_MODE_CONTROL(dev, vsec, dest) \  argument
54 	pci_read_config_byte(dev, vsec + 0xa, dest)
55 #define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \  argument
56 	pci_write_config_byte(dev, vsec + 0xa, val)
63 #define CXL_READ_VSEC_PSL_REVISION(dev, vsec, dest) \  argument
64 	pci_read_config_word(dev, vsec + 0xc, dest)
65 #define CXL_READ_VSEC_CAIA_MINOR(dev, vsec, dest) \  argument
66 	pci_read_config_byte(dev, vsec + 0xe, dest)
67 #define CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, dest) \  argument
68 	pci_read_config_byte(dev, vsec + 0xf, dest)
69 #define CXL_READ_VSEC_BASE_IMAGE(dev, vsec, dest) \  argument
70 	pci_read_config_word(dev, vsec + 0x10, dest)
72 #define CXL_READ_VSEC_IMAGE_STATE(dev, vsec, dest) \  argument
73 	pci_read_config_byte(dev, vsec + 0x13, dest)
74 #define CXL_WRITE_VSEC_IMAGE_STATE(dev, vsec, val) \  argument
75 	pci_write_config_byte(dev, vsec + 0x13, val)
80 #define CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, dest) \  argument
81 	pci_read_config_dword(dev, vsec + 0x20, dest)
82 #define CXL_READ_VSEC_AFU_DESC_SIZE(dev, vsec, dest) \  argument
83 	pci_read_config_dword(dev, vsec + 0x24, dest)
84 #define CXL_READ_VSEC_PS_OFF(dev, vsec, dest) \  argument
85 	pci_read_config_dword(dev, vsec + 0x28, dest)
86 #define CXL_READ_VSEC_PS_SIZE(dev, vsec, dest) \  argument
87 	pci_read_config_dword(dev, vsec + 0x2c, dest)
150 static inline resource_size_t p1_base(struct pci_dev *dev)  in p1_base()  argument
152 	return pci_resource_start(dev, 2);  in p1_base()
155 static inline resource_size_t p1_size(struct pci_dev *dev)  in p1_size()  argument
157 	return pci_resource_len(dev, 2);  in p1_size()
160 static inline resource_size_t p2_base(struct pci_dev *dev)  in p2_base()  argument
162 	return pci_resource_start(dev, 0);  in p2_base()
165 static inline resource_size_t p2_size(struct pci_dev *dev)  in p2_size()  argument
167 	return pci_resource_len(dev, 0);  in p2_size()
170 static int find_cxl_vsec(struct pci_dev *dev)  in find_cxl_vsec()  argument
175 	while ((vsec = pci_find_next_ext_capability(dev, vsec, PCI_EXT_CAP_ID_VNDR))) {  in find_cxl_vsec()
176 		pci_read_config_word(dev, vsec + 0x4, &val);  in find_cxl_vsec()
184 static void dump_cxl_config_space(struct pci_dev *dev)  in dump_cxl_config_space()  argument
189 	dev_info(&dev->dev, "dump_cxl_config_space\n");  in dump_cxl_config_space()
191 	pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &val);  in dump_cxl_config_space()
192 	dev_info(&dev->dev, "BAR0: %#.8x\n", val);  in dump_cxl_config_space()
193 	pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val);  in dump_cxl_config_space()
194 	dev_info(&dev->dev, "BAR1: %#.8x\n", val);  in dump_cxl_config_space()
195 	pci_read_config_dword(dev, PCI_BASE_ADDRESS_2, &val);  in dump_cxl_config_space()
196 	dev_info(&dev->dev, "BAR2: %#.8x\n", val);  in dump_cxl_config_space()
197 	pci_read_config_dword(dev, PCI_BASE_ADDRESS_3, &val);  in dump_cxl_config_space()
198 	dev_info(&dev->dev, "BAR3: %#.8x\n", val);  in dump_cxl_config_space()
199 	pci_read_config_dword(dev, PCI_BASE_ADDRESS_4, &val);  in dump_cxl_config_space()
200 	dev_info(&dev->dev, "BAR4: %#.8x\n", val);  in dump_cxl_config_space()
201 	pci_read_config_dword(dev, PCI_BASE_ADDRESS_5, &val);  in dump_cxl_config_space()
202 	dev_info(&dev->dev, "BAR5: %#.8x\n", val);  in dump_cxl_config_space()
204 	dev_info(&dev->dev, "p1 regs: %#llx, len: %#llx\n",  in dump_cxl_config_space()
205 		p1_base(dev), p1_size(dev));  in dump_cxl_config_space()
206 	dev_info(&dev->dev, "p2 regs: %#llx, len: %#llx\n",  in dump_cxl_config_space()
207 		p1_base(dev), p2_size(dev));  in dump_cxl_config_space()
208 	dev_info(&dev->dev, "BAR 4/5: %#llx, len: %#llx\n",  in dump_cxl_config_space()
209 		pci_resource_start(dev, 4), pci_resource_len(dev, 4));  in dump_cxl_config_space()
211 	if (!(vsec = find_cxl_vsec(dev)))  in dump_cxl_config_space()
215 	dev_info(&dev->dev, "cxl vsec: %30s: %#x\n", name, what)  in dump_cxl_config_space()
217 	pci_read_config_dword(dev, vsec + 0x0, &val);  in dump_cxl_config_space()
221 	pci_read_config_dword(dev, vsec + 0x4, &val);  in dump_cxl_config_space()
225 	pci_read_config_dword(dev, vsec + 0x8, &val);  in dump_cxl_config_space()
230 	pci_read_config_dword(dev, vsec + 0xc, &val);  in dump_cxl_config_space()
233 	pci_read_config_dword(dev, vsec + 0x10, &val);  in dump_cxl_config_space()
240 	pci_read_config_dword(dev, vsec + 0x14, &val);  in dump_cxl_config_space()
242 	pci_read_config_dword(dev, vsec + 0x18, &val);  in dump_cxl_config_space()
244 	pci_read_config_dword(dev, vsec + 0x1c, &val);  in dump_cxl_config_space()
247 	pci_read_config_dword(dev, vsec + 0x20, &val);  in dump_cxl_config_space()
249 	pci_read_config_dword(dev, vsec + 0x24, &val);  in dump_cxl_config_space()
251 	pci_read_config_dword(dev, vsec + 0x28, &val);  in dump_cxl_config_space()
253 	pci_read_config_dword(dev, vsec + 0x2c, &val);  in dump_cxl_config_space()
256 	pci_read_config_dword(dev, vsec + 0x30, &val);  in dump_cxl_config_space()
258 	pci_read_config_dword(dev, vsec + 0x34, &val);  in dump_cxl_config_space()
260 	pci_read_config_dword(dev, vsec + 0x38, &val);  in dump_cxl_config_space()
262 	pci_read_config_dword(dev, vsec + 0x3c, &val);  in dump_cxl_config_space()
265 	pci_read_config_dword(dev, vsec + 0x40, &val);  in dump_cxl_config_space()
267 	pci_read_config_dword(dev, vsec + 0x44, &val);  in dump_cxl_config_space()
270 	pci_read_config_dword(dev, vsec + 0x48, &val);  in dump_cxl_config_space()
272 	pci_read_config_dword(dev, vsec + 0x4c, &val);  in dump_cxl_config_space()
275 	pci_read_config_dword(dev, vsec + 0x50, &val);  in dump_cxl_config_space()
277 	pci_read_config_dword(dev, vsec + 0x54, &val);  in dump_cxl_config_space()
279 	pci_read_config_dword(dev, vsec + 0x58, &val);  in dump_cxl_config_space()
281 	pci_read_config_dword(dev, vsec + 0x58, &val);  in dump_cxl_config_space()
292 	dev_info(&afu->dev, "afu desc: %30s: %#llx\n", name, what)  in dump_afu_descriptor()
331 static int init_implementation_adapter_regs(struct cxl *adapter, struct pci_dev *dev)  in init_implementation_adapter_regs()  argument
338 	if (!(np = pnv_pci_get_phb_node(dev)))  in init_implementation_adapter_regs()
378 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_setup_irq()  local
380 	return pnv_cxl_ioda_msi_setup(dev, hwirq, virq);  in cxl_setup_irq()
385 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_update_image_control()  local
390 	if (!(vsec = find_cxl_vsec(dev))) {  in cxl_update_image_control()
391 		dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");  in cxl_update_image_control()
395 	if ((rc = CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state))) {  in cxl_update_image_control()
396 		dev_err(&dev->dev, "failed to read image state: %i\n", rc);  in cxl_update_image_control()
410 	if ((rc = CXL_WRITE_VSEC_IMAGE_STATE(dev, vsec, image_state))) {  in cxl_update_image_control()
411 		dev_err(&dev->dev, "failed to update image control: %i\n", rc);  in cxl_update_image_control()
420 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_alloc_one_irq()  local
422 	return pnv_cxl_alloc_hwirqs(dev, 1);  in cxl_alloc_one_irq()
427 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_release_one_irq()  local
429 	return pnv_cxl_release_hwirqs(dev, hwirq, 1);  in cxl_release_one_irq()
434 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_alloc_irq_ranges()  local
436 	return pnv_cxl_alloc_hwirq_ranges(irqs, dev, num);  in cxl_alloc_irq_ranges()
441 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_release_irq_ranges()  local
443 	pnv_cxl_release_hwirq_ranges(irqs, dev);  in cxl_release_irq_ranges()
446 static int setup_cxl_bars(struct pci_dev *dev)  in setup_cxl_bars()  argument
449 	if ((p1_base(dev) < 0x100000000ULL) ||  in setup_cxl_bars()
450 	    (p2_base(dev) < 0x100000000ULL)) {  in setup_cxl_bars()
451 		dev_err(&dev->dev, "ABORTING: M32 BAR assignment incompatible with CXL\n");  in setup_cxl_bars()
460 	pci_write_config_dword(dev, PCI_BASE_ADDRESS_4, 0x00000000);  in setup_cxl_bars()
461 	pci_write_config_dword(dev, PCI_BASE_ADDRESS_5, 0x00020000);  in setup_cxl_bars()
467 static int switch_card_to_cxl(struct pci_dev *dev)  in switch_card_to_cxl()  argument
473 	dev_info(&dev->dev, "switch card to CXL\n");  in switch_card_to_cxl()
475 	if (!(vsec = find_cxl_vsec(dev))) {  in switch_card_to_cxl()
476 		dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");  in switch_card_to_cxl()
480 	if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) {  in switch_card_to_cxl()
481 		dev_err(&dev->dev, "failed to read current mode control: %i", rc);  in switch_card_to_cxl()
486 	if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) {  in switch_card_to_cxl()
487 		dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc);  in switch_card_to_cxl()
500 static int cxl_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in cxl_map_slice_regs()  argument
506 	p1n_base = p1_base(dev) + 0x10000 + (afu->slice * p1n_size);  in cxl_map_slice_regs()
507 	p2n_base = p2_base(dev) + (afu->slice * p2n_size);  in cxl_map_slice_regs()
508 	afu->psn_phys = p2_base(dev) + (adapter->ps_off + (afu->slice * adapter->ps_size));  in cxl_map_slice_regs()
509 	afu_desc = p2_base(dev) + adapter->afu_desc_off + (afu->slice * adapter->afu_desc_size);  in cxl_map_slice_regs()
526 	dev_err(&afu->dev, "Error mapping AFU MMIO regions\n");  in cxl_map_slice_regs()
538 static void cxl_release_afu(struct device *dev)  in cxl_release_afu()  argument
540 	struct cxl_afu *afu = to_cxl_afu(dev);  in cxl_release_afu()
555 	afu->dev.parent = &adapter->dev;  in cxl_alloc_afu()
556 	afu->dev.release = cxl_release_afu;  in cxl_alloc_afu()
605 		dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n");  in cxl_afu_descriptor_looks_ok()
610 		dev_warn(&afu->dev, "AFU uses < PAGE_SIZE per-process PSA!");  in cxl_afu_descriptor_looks_ok()
614 			dev_err(&afu->dev, "ABORTING: AFU configuration record %i is invalid\n", i);  in cxl_afu_descriptor_looks_ok()
633 		dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#.16llx\n", reg);  in sanitise_afu_regs()
654 		dev_warn(&afu->dev, "AFU had pending DSISR: %#.16llx\n", reg);  in sanitise_afu_regs()
663 			dev_warn(&afu->dev, "AFU had pending SERR: %#.16llx\n", reg);  in sanitise_afu_regs()
668 		dev_warn(&afu->dev, "AFU had pending error status: %#.16llx\n", reg);  in sanitise_afu_regs()
675 static int cxl_init_afu(struct cxl *adapter, int slice, struct pci_dev *dev)  in cxl_init_afu()  argument
684 	if ((rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice)))  in cxl_init_afu()
687 	if ((rc = cxl_map_slice_regs(afu, adapter, dev)))  in cxl_init_afu()
739 	device_unregister(&afu->dev);  in cxl_init_afu()
774 	device_unregister(&afu->dev);  in cxl_remove_afu()
779 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_reset()  local
782 	dev_info(&dev->dev, "CXL reset\n");  in cxl_reset()
787 	if ((rc = pci_set_pcie_reset_state(dev, pcie_warm_reset))) {  in cxl_reset()
788 		dev_err(&dev->dev, "cxl: pcie_warm_reset failed\n");  in cxl_reset()
795 static int cxl_map_adapter_regs(struct cxl *adapter, struct pci_dev *dev)  in cxl_map_adapter_regs()  argument
797 	if (pci_request_region(dev, 2, "priv 2 regs"))  in cxl_map_adapter_regs()
799 	if (pci_request_region(dev, 0, "priv 1 regs"))  in cxl_map_adapter_regs()
803 			p1_base(dev), p1_size(dev), p2_base(dev), p2_size(dev));  in cxl_map_adapter_regs()
805 	if (!(adapter->p1_mmio = ioremap(p1_base(dev), p1_size(dev))))  in cxl_map_adapter_regs()
808 	if (!(adapter->p2_mmio = ioremap(p2_base(dev), p2_size(dev))))  in cxl_map_adapter_regs()
817 	pci_release_region(dev, 0);  in cxl_map_adapter_regs()
819 	pci_release_region(dev, 2);  in cxl_map_adapter_regs()
832 static int cxl_read_vsec(struct cxl *adapter, struct pci_dev *dev)  in cxl_read_vsec()  argument
840 	if (!(vsec = find_cxl_vsec(dev))) {  in cxl_read_vsec()
841 		dev_err(&adapter->dev, "ABORTING: CXL VSEC not found!\n");  in cxl_read_vsec()
845 	CXL_READ_VSEC_LENGTH(dev, vsec, &vseclen);  in cxl_read_vsec()
851 	CXL_READ_VSEC_STATUS(dev, vsec, &adapter->vsec_status);  in cxl_read_vsec()
852 	CXL_READ_VSEC_PSL_REVISION(dev, vsec, &adapter->psl_rev);  in cxl_read_vsec()
853 	CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, &adapter->caia_major);  in cxl_read_vsec()
854 	CXL_READ_VSEC_CAIA_MINOR(dev, vsec, &adapter->caia_minor);  in cxl_read_vsec()
855 	CXL_READ_VSEC_BASE_IMAGE(dev, vsec, &adapter->base_image);  in cxl_read_vsec()
856 	CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state);  in cxl_read_vsec()
861 	CXL_READ_VSEC_NAFUS(dev, vsec, &adapter->slices);  in cxl_read_vsec()
862 	CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, &afu_desc_off);  in cxl_read_vsec()
863 	CXL_READ_VSEC_AFU_DESC_SIZE(dev, vsec, &afu_desc_size);  in cxl_read_vsec()
864 	CXL_READ_VSEC_PS_OFF(dev, vsec, &ps_off);  in cxl_read_vsec()
865 	CXL_READ_VSEC_PS_SIZE(dev, vsec, &ps_size);  in cxl_read_vsec()
875 	adapter->user_irqs = pnv_cxl_get_irq_count(dev) - 1 - 2*adapter->slices;  in cxl_read_vsec()
880 static int cxl_vsec_looks_ok(struct cxl *adapter, struct pci_dev *dev)  in cxl_vsec_looks_ok()  argument
886 		dev_err(&adapter->dev, "ABORTING: CXL requires unsupported features\n");  in cxl_vsec_looks_ok()
893 		dev_err(&adapter->dev, "ABORTING: Device has no AFUs\n");  in cxl_vsec_looks_ok()
898 		dev_err(&adapter->dev, "ABORTING: VSEC shows no AFU descriptors\n");  in cxl_vsec_looks_ok()
902 	if (adapter->ps_size > p2_size(dev) - adapter->ps_off) {  in cxl_vsec_looks_ok()
903 		dev_err(&adapter->dev, "ABORTING: Problem state size larger than "  in cxl_vsec_looks_ok()
905 			 adapter->ps_size, p2_size(dev) - adapter->ps_off);  in cxl_vsec_looks_ok()
912 static void cxl_release_adapter(struct device *dev)  in cxl_release_adapter()  argument
914 	struct cxl *adapter = to_cxl_adapter(dev);  in cxl_release_adapter()
921 static struct cxl *cxl_alloc_adapter(struct pci_dev *dev)  in cxl_alloc_adapter()  argument
928 	adapter->dev.parent = &dev->dev;  in cxl_alloc_adapter()
929 	adapter->dev.release = cxl_release_adapter;  in cxl_alloc_adapter()
930 	pci_set_drvdata(dev, adapter);  in cxl_alloc_adapter()
942 static struct cxl *cxl_init_adapter(struct pci_dev *dev)  in cxl_init_adapter()  argument
949 	if (!(adapter = cxl_alloc_adapter(dev)))  in cxl_init_adapter()
952 	if ((rc = switch_card_to_cxl(dev)))  in cxl_init_adapter()
958 	if ((rc = dev_set_name(&adapter->dev, "card%i", adapter->adapter_num)))  in cxl_init_adapter()
961 	if ((rc = cxl_read_vsec(adapter, dev)))  in cxl_init_adapter()
964 	if ((rc = cxl_vsec_looks_ok(adapter, dev)))  in cxl_init_adapter()
970 	if ((rc = cxl_map_adapter_regs(adapter, dev)))  in cxl_init_adapter()
976 	if ((rc = init_implementation_adapter_regs(adapter, dev)))  in cxl_init_adapter()
979 	if ((rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_CAPI)))  in cxl_init_adapter()
984 	if ((rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON))) {  in cxl_init_adapter()
1007 	device_unregister(&adapter->dev);  in cxl_init_adapter()
1023 	struct pci_dev *pdev = to_pci_dev(adapter->dev.parent);  in cxl_remove_adapter()
1033 	device_unregister(&adapter->dev);  in cxl_remove_adapter()
1040 static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)  in cxl_probe()  argument
1047 		dump_cxl_config_space(dev);  in cxl_probe()
1049 	if ((rc = setup_cxl_bars(dev)))  in cxl_probe()
1052 	if ((rc = pci_enable_device(dev))) {  in cxl_probe()
1053 		dev_err(&dev->dev, "pci_enable_device failed: %i\n", rc);  in cxl_probe()
1057 	adapter = cxl_init_adapter(dev);  in cxl_probe()
1059 		dev_err(&dev->dev, "cxl_init_adapter failed: %li\n", PTR_ERR(adapter));  in cxl_probe()
1064 		if ((rc = cxl_init_afu(adapter, slice, dev)))  in cxl_probe()
1065 			dev_err(&dev->dev, "AFU %i failed to initialise: %i\n", slice, rc);  in cxl_probe()
1071 static void cxl_remove(struct pci_dev *dev)  in cxl_remove()  argument
1073 	struct cxl *adapter = pci_get_drvdata(dev);  in cxl_remove()
1076 	dev_warn(&dev->dev, "pci remove\n");  in cxl_remove()