93 #define AFUD_READ(afu, off)		in_be64(afu->afu_desc_mmio + off)  argument
94 #define AFUD_READ_LE(afu, off)		in_le64(afu->afu_desc_mmio + off)  argument
98 #define AFUD_READ_INFO(afu)		AFUD_READ(afu, 0x0)  argument
107 #define AFUD_READ_CR(afu)		AFUD_READ(afu, 0x20)  argument
109 #define AFUD_READ_CR_OFF(afu)		AFUD_READ(afu, 0x28)  argument
110 #define AFUD_READ_PPPSA(afu)		AFUD_READ(afu, 0x30)  argument
114 #define AFUD_READ_PPPSA_OFF(afu)	AFUD_READ(afu, 0x38)  argument
115 #define AFUD_READ_EB(afu)		AFUD_READ(afu, 0x40)  argument
117 #define AFUD_READ_EB_OFF(afu)		AFUD_READ(afu, 0x48)  argument
119 u16 cxl_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off)  in cxl_afu_cr_read16()  argument
124 	val = cxl_afu_cr_read32(afu, cr, aligned_off);  in cxl_afu_cr_read16()
128 u8 cxl_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off)  in cxl_afu_cr_read8()  argument
133 	val = cxl_afu_cr_read32(afu, cr, aligned_off);  in cxl_afu_cr_read8()
289 static void dump_afu_descriptor(struct cxl_afu *afu)  in dump_afu_descriptor()  argument
295 	dev_info(&afu->dev, "afu desc: %30s: %#llx\n", name, what)  in dump_afu_descriptor()
297 	val = AFUD_READ_INFO(afu);  in dump_afu_descriptor()
304 	val = AFUD_READ(afu, 0x8);  in dump_afu_descriptor()
306 	val = AFUD_READ(afu, 0x10);  in dump_afu_descriptor()
308 	val = AFUD_READ(afu, 0x18);  in dump_afu_descriptor()
311 	val = AFUD_READ_CR(afu);  in dump_afu_descriptor()
316 	val = AFUD_READ_CR_OFF(afu);  in dump_afu_descriptor()
320 	val = AFUD_READ_PPPSA(afu);  in dump_afu_descriptor()
324 	val = AFUD_READ_PPPSA_OFF(afu);  in dump_afu_descriptor()
327 	val = AFUD_READ_EB(afu);  in dump_afu_descriptor()
331 	val = AFUD_READ_EB_OFF(afu);  in dump_afu_descriptor()
335 		val = AFUD_READ_LE(afu, afu_cr_off + i * afu_cr_len);  in dump_afu_descriptor()
422 static int init_implementation_afu_regs(struct cxl_afu *afu)  in init_implementation_afu_regs()  argument
425 	cxl_p1n_write(afu, CXL_PSL_APCALLOC_A, 0xFFFFFFFEFEFEFEFEULL);  in init_implementation_afu_regs()
427 	cxl_p1n_write(afu, CXL_PSL_COALLOC_A, 0xFF000000FEFEFEFEULL);  in init_implementation_afu_regs()
429 	cxl_p1n_write(afu, CXL_PSL_SLICE_TRACE, 0x0000FFFF00000000ULL);  in init_implementation_afu_regs()
430 	cxl_p1n_write(afu, CXL_PSL_RXCTL_A, CXL_PSL_RXCTL_AFUHP_4S);  in init_implementation_afu_regs()
560 static int cxl_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in cxl_map_slice_regs()  argument
566 	p1n_base = p1_base(dev) + 0x10000 + (afu->slice * p1n_size);  in cxl_map_slice_regs()
567 	p2n_base = p2_base(dev) + (afu->slice * p2n_size);  in cxl_map_slice_regs()
568 	afu->psn_phys = p2_base(dev) + (adapter->ps_off + (afu->slice * adapter->ps_size));  in cxl_map_slice_regs()
569 	afu_desc = p2_base(dev) + adapter->afu_desc_off + (afu->slice * adapter->afu_desc_size);  in cxl_map_slice_regs()
571 	if (!(afu->p1n_mmio = ioremap(p1n_base, p1n_size)))  in cxl_map_slice_regs()
573 	if (!(afu->p2n_mmio = ioremap(p2n_base, p2n_size)))  in cxl_map_slice_regs()
576 		if (!(afu->afu_desc_mmio = ioremap(afu_desc, adapter->afu_desc_size)))  in cxl_map_slice_regs()
582 	iounmap(afu->p2n_mmio);  in cxl_map_slice_regs()
584 	iounmap(afu->p1n_mmio);  in cxl_map_slice_regs()
586 	dev_err(&afu->dev, "Error mapping AFU MMIO regions\n");  in cxl_map_slice_regs()
590 static void cxl_unmap_slice_regs(struct cxl_afu *afu)  in cxl_unmap_slice_regs()  argument
592 	if (afu->p2n_mmio) {  in cxl_unmap_slice_regs()
593 		iounmap(afu->p2n_mmio);  in cxl_unmap_slice_regs()
594 		afu->p2n_mmio = NULL;  in cxl_unmap_slice_regs()
596 	if (afu->p1n_mmio) {  in cxl_unmap_slice_regs()
597 		iounmap(afu->p1n_mmio);  in cxl_unmap_slice_regs()
598 		afu->p1n_mmio = NULL;  in cxl_unmap_slice_regs()
600 	if (afu->afu_desc_mmio) {  in cxl_unmap_slice_regs()
601 		iounmap(afu->afu_desc_mmio);  in cxl_unmap_slice_regs()
602 		afu->afu_desc_mmio = NULL;  in cxl_unmap_slice_regs()
608 	struct cxl_afu *afu = to_cxl_afu(dev);  in cxl_release_afu()  local
612 	idr_destroy(&afu->contexts_idr);  in cxl_release_afu()
613 	cxl_release_spa(afu);  in cxl_release_afu()
615 	kfree(afu);  in cxl_release_afu()
620 	struct cxl_afu *afu;  in cxl_alloc_afu()  local
622 	if (!(afu = kzalloc(sizeof(struct cxl_afu), GFP_KERNEL)))  in cxl_alloc_afu()
625 	afu->adapter = adapter;  in cxl_alloc_afu()
626 	afu->dev.parent = &adapter->dev;  in cxl_alloc_afu()
627 	afu->dev.release = cxl_release_afu;  in cxl_alloc_afu()
628 	afu->slice = slice;  in cxl_alloc_afu()
629 	idr_init(&afu->contexts_idr);  in cxl_alloc_afu()
630 	mutex_init(&afu->contexts_lock);  in cxl_alloc_afu()
631 	spin_lock_init(&afu->afu_cntl_lock);  in cxl_alloc_afu()
632 	mutex_init(&afu->spa_mutex);  in cxl_alloc_afu()
634 	afu->prefault_mode = CXL_PREFAULT_NONE;  in cxl_alloc_afu()
635 	afu->irqs_max = afu->adapter->user_irqs;  in cxl_alloc_afu()
637 	return afu;  in cxl_alloc_afu()
641 static int cxl_read_afu_descriptor(struct cxl_afu *afu)  in cxl_read_afu_descriptor()  argument
645 	val = AFUD_READ_INFO(afu);  in cxl_read_afu_descriptor()
646 	afu->pp_irqs = AFUD_NUM_INTS_PER_PROC(val);  in cxl_read_afu_descriptor()
647 	afu->max_procs_virtualised = AFUD_NUM_PROCS(val);  in cxl_read_afu_descriptor()
648 	afu->crs_num = AFUD_NUM_CRS(val);  in cxl_read_afu_descriptor()
651 		afu->modes_supported |= CXL_MODE_DIRECTED;  in cxl_read_afu_descriptor()
653 		afu->modes_supported |= CXL_MODE_DEDICATED;  in cxl_read_afu_descriptor()
655 		afu->modes_supported |= CXL_MODE_TIME_SLICED;  in cxl_read_afu_descriptor()
657 	val = AFUD_READ_PPPSA(afu);  in cxl_read_afu_descriptor()
658 	afu->pp_size = AFUD_PPPSA_LEN(val) * 4096;  in cxl_read_afu_descriptor()
659 	afu->psa = AFUD_PPPSA_PSA(val);  in cxl_read_afu_descriptor()
660 	if ((afu->pp_psa = AFUD_PPPSA_PP(val)))  in cxl_read_afu_descriptor()
661 		afu->pp_offset = AFUD_READ_PPPSA_OFF(afu);  in cxl_read_afu_descriptor()
663 	val = AFUD_READ_CR(afu);  in cxl_read_afu_descriptor()
664 	afu->crs_len = AFUD_CR_LEN(val) * 256;  in cxl_read_afu_descriptor()
665 	afu->crs_offset = AFUD_READ_CR_OFF(afu);  in cxl_read_afu_descriptor()
669 	afu->eb_len = AFUD_EB_LEN(AFUD_READ_EB(afu)) * 4096;  in cxl_read_afu_descriptor()
670 	afu->eb_offset = AFUD_READ_EB_OFF(afu);  in cxl_read_afu_descriptor()
673 	if (EXTRACT_PPC_BITS(afu->eb_offset, 0, 11) != 0) {  in cxl_read_afu_descriptor()
674 		dev_warn(&afu->dev,  in cxl_read_afu_descriptor()
676 			 afu->eb_offset);  in cxl_read_afu_descriptor()
677 		dev_info(&afu->dev,  in cxl_read_afu_descriptor()
680 		afu->eb_len = 0;  in cxl_read_afu_descriptor()
686 static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu)  in cxl_afu_descriptor_looks_ok()  argument
690 	if (afu->psa && afu->adapter->ps_size <  in cxl_afu_descriptor_looks_ok()
691 			(afu->pp_offset + afu->pp_size*afu->max_procs_virtualised)) {  in cxl_afu_descriptor_looks_ok()
692 		dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n");  in cxl_afu_descriptor_looks_ok()
696 	if (afu->pp_psa && (afu->pp_size < PAGE_SIZE))  in cxl_afu_descriptor_looks_ok()
697 		dev_warn(&afu->dev, "AFU uses < PAGE_SIZE per-process PSA!");  in cxl_afu_descriptor_looks_ok()
699 	for (i = 0; i < afu->crs_num; i++) {  in cxl_afu_descriptor_looks_ok()
700 		if ((cxl_afu_cr_read32(afu, i, 0) == 0)) {  in cxl_afu_descriptor_looks_ok()
701 			dev_err(&afu->dev, "ABORTING: AFU configuration record %i is invalid\n", i);  in cxl_afu_descriptor_looks_ok()
709 static int sanitise_afu_regs(struct cxl_afu *afu)  in sanitise_afu_regs()  argument
718 	reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);  in sanitise_afu_regs()
720 		dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#016llx\n", reg);  in sanitise_afu_regs()
721 		if (__cxl_afu_reset(afu))  in sanitise_afu_regs()
723 		if (cxl_afu_disable(afu))  in sanitise_afu_regs()
725 		if (cxl_psl_purge(afu))  in sanitise_afu_regs()
728 	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);  in sanitise_afu_regs()
729 	cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, 0x0000000000000000);  in sanitise_afu_regs()
730 	cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 0x0000000000000000);  in sanitise_afu_regs()
731 	cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);  in sanitise_afu_regs()
732 	cxl_p1n_write(afu, CXL_PSL_SPOffset_An, 0x0000000000000000);  in sanitise_afu_regs()
733 	cxl_p1n_write(afu, CXL_HAURP_An, 0x0000000000000000);  in sanitise_afu_regs()
734 	cxl_p2n_write(afu, CXL_CSRP_An, 0x0000000000000000);  in sanitise_afu_regs()
735 	cxl_p2n_write(afu, CXL_AURP1_An, 0x0000000000000000);  in sanitise_afu_regs()
736 	cxl_p2n_write(afu, CXL_AURP0_An, 0x0000000000000000);  in sanitise_afu_regs()
737 	cxl_p2n_write(afu, CXL_SSTP1_An, 0x0000000000000000);  in sanitise_afu_regs()
738 	cxl_p2n_write(afu, CXL_SSTP0_An, 0x0000000000000000);  in sanitise_afu_regs()
739 	reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);  in sanitise_afu_regs()
741 		dev_warn(&afu->dev, "AFU had pending DSISR: %#016llx\n", reg);  in sanitise_afu_regs()
743 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);  in sanitise_afu_regs()
745 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);  in sanitise_afu_regs()
747 	reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);  in sanitise_afu_regs()
750 			dev_warn(&afu->dev, "AFU had pending SERR: %#016llx\n", reg);  in sanitise_afu_regs()
751 		cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);  in sanitise_afu_regs()
753 	reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);  in sanitise_afu_regs()
755 		dev_warn(&afu->dev, "AFU had pending error status: %#016llx\n", reg);  in sanitise_afu_regs()
756 		cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);  in sanitise_afu_regs()
769 ssize_t cxl_afu_read_err_buffer(struct cxl_afu *afu, char *buf,  in cxl_afu_read_err_buffer()  argument
775 	const void __iomem *ebuf = afu->afu_desc_mmio + afu->eb_offset;  in cxl_afu_read_err_buffer()
777 	if (count == 0 || off < 0 || (size_t)off >= afu->eb_len)  in cxl_afu_read_err_buffer()
781 	count = min((size_t)(afu->eb_len - off), count);  in cxl_afu_read_err_buffer()
806 static int cxl_configure_afu(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in cxl_configure_afu()  argument
810 	if ((rc = cxl_map_slice_regs(afu, adapter, dev)))  in cxl_configure_afu()
813 	if ((rc = sanitise_afu_regs(afu)))  in cxl_configure_afu()
817 	if ((rc = __cxl_afu_reset(afu)))  in cxl_configure_afu()
821 		dump_afu_descriptor(afu);  in cxl_configure_afu()
823 	if ((rc = cxl_read_afu_descriptor(afu)))  in cxl_configure_afu()
826 	if ((rc = cxl_afu_descriptor_looks_ok(afu)))  in cxl_configure_afu()
829 	if ((rc = init_implementation_afu_regs(afu)))  in cxl_configure_afu()
832 	if ((rc = cxl_register_serr_irq(afu)))  in cxl_configure_afu()
835 	if ((rc = cxl_register_psl_irq(afu)))  in cxl_configure_afu()
841 	cxl_release_serr_irq(afu);  in cxl_configure_afu()
843 	cxl_unmap_slice_regs(afu);  in cxl_configure_afu()
847 static void cxl_deconfigure_afu(struct cxl_afu *afu)  in cxl_deconfigure_afu()  argument
849 	cxl_release_psl_irq(afu);  in cxl_deconfigure_afu()
850 	cxl_release_serr_irq(afu);  in cxl_deconfigure_afu()
851 	cxl_unmap_slice_regs(afu);  in cxl_deconfigure_afu()
856 	struct cxl_afu *afu;  in cxl_init_afu()  local
859 	afu = cxl_alloc_afu(adapter, slice);  in cxl_init_afu()
860 	if (!afu)  in cxl_init_afu()
863 	rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice);  in cxl_init_afu()
867 	rc = cxl_configure_afu(afu, adapter, dev);  in cxl_init_afu()
872 	cxl_debugfs_afu_add(afu);  in cxl_init_afu()
878 	if ((rc = cxl_register_afu(afu)))  in cxl_init_afu()
881 	if ((rc = cxl_sysfs_afu_add(afu)))  in cxl_init_afu()
884 	adapter->afu[afu->slice] = afu;  in cxl_init_afu()
886 	if ((rc = cxl_pci_vphb_add(afu)))  in cxl_init_afu()
887 		dev_info(&afu->dev, "Can't register vPHB\n");  in cxl_init_afu()
892 	cxl_deconfigure_afu(afu);  in cxl_init_afu()
893 	cxl_debugfs_afu_remove(afu);  in cxl_init_afu()
894 	device_unregister(&afu->dev);  in cxl_init_afu()
898 	kfree(afu);  in cxl_init_afu()
903 static void cxl_remove_afu(struct cxl_afu *afu)  in cxl_remove_afu()  argument
907 	if (!afu)  in cxl_remove_afu()
910 	cxl_sysfs_afu_remove(afu);  in cxl_remove_afu()
911 	cxl_debugfs_afu_remove(afu);  in cxl_remove_afu()
913 	spin_lock(&afu->adapter->afu_list_lock);  in cxl_remove_afu()
914 	afu->adapter->afu[afu->slice] = NULL;  in cxl_remove_afu()
915 	spin_unlock(&afu->adapter->afu_list_lock);  in cxl_remove_afu()
917 	cxl_context_detach_all(afu);  in cxl_remove_afu()
918 	cxl_afu_deactivate_mode(afu);  in cxl_remove_afu()
920 	cxl_deconfigure_afu(afu);  in cxl_remove_afu()
921 	device_unregister(&afu->dev);  in cxl_remove_afu()
1295 		rc = cxl_afu_select_best_mode(adapter->afu[slice]);  in cxl_probe()
1306 	struct cxl_afu *afu;  in cxl_remove()  local
1314 		afu = adapter->afu[i];  in cxl_remove()
1315 		cxl_pci_vphb_remove(afu);  in cxl_remove()
1316 		cxl_remove_afu(afu);  in cxl_remove()
1321 static pci_ers_result_t cxl_vphb_error_detected(struct cxl_afu *afu,  in cxl_vphb_error_detected()  argument
1331 	list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {  in cxl_vphb_error_detected()
1354 	struct cxl_afu *afu;  in cxl_pci_error_detected()  local
1370 			afu = adapter->afu[i];  in cxl_pci_error_detected()
1371 			cxl_vphb_error_detected(afu, state);  in cxl_pci_error_detected()
1455 		afu = adapter->afu[i];  in cxl_pci_error_detected()
1457 		result = cxl_vphb_error_detected(afu, state);  in cxl_pci_error_detected()
1463 		cxl_context_detach_all(afu);  in cxl_pci_error_detected()
1464 		cxl_afu_deactivate_mode(afu);  in cxl_pci_error_detected()
1465 		cxl_deconfigure_afu(afu);  in cxl_pci_error_detected()
1475 	struct cxl_afu *afu;  in cxl_pci_slot_reset()  local
1486 		afu = adapter->afu[i];  in cxl_pci_slot_reset()
1488 		if (cxl_configure_afu(afu, adapter, pdev))  in cxl_pci_slot_reset()
1491 		if (cxl_afu_select_best_mode(afu))  in cxl_pci_slot_reset()
1494 		cxl_pci_vphb_reconfigure(afu);  in cxl_pci_slot_reset()
1496 		list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {  in cxl_pci_slot_reset()
1511 			if (cxl_afu_check_and_enable(afu))  in cxl_pci_slot_reset()
1547 	struct cxl_afu *afu;  in cxl_pci_resume()  local
1556 		afu = adapter->afu[i];  in cxl_pci_resume()
1558 		list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {  in cxl_pci_resume()