92 #define AFUD_READ(afu, off)		in_be64(afu->afu_desc_mmio + off)  argument
96 #define AFUD_READ_INFO(afu)		AFUD_READ(afu, 0x0)  argument
105 #define AFUD_READ_CR(afu)		AFUD_READ(afu, 0x20)  argument
107 #define AFUD_READ_CR_OFF(afu)		AFUD_READ(afu, 0x28)  argument
108 #define AFUD_READ_PPPSA(afu)		AFUD_READ(afu, 0x30)  argument
112 #define AFUD_READ_PPPSA_OFF(afu)	AFUD_READ(afu, 0x38)  argument
113 #define AFUD_READ_EB(afu)		AFUD_READ(afu, 0x40)  argument
115 #define AFUD_READ_EB_OFF(afu)		AFUD_READ(afu, 0x48)  argument
117 u16 cxl_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off)  in cxl_afu_cr_read16()  argument
122 	val = cxl_afu_cr_read32(afu, cr, aligned_off);  in cxl_afu_cr_read16()
126 u8 cxl_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off)  in cxl_afu_cr_read8()  argument
131 	val = cxl_afu_cr_read32(afu, cr, aligned_off);  in cxl_afu_cr_read8()
287 static void dump_afu_descriptor(struct cxl_afu *afu)  in dump_afu_descriptor()  argument
292 	dev_info(&afu->dev, "afu desc: %30s: %#llx\n", name, what)  in dump_afu_descriptor()
294 	val = AFUD_READ_INFO(afu);  in dump_afu_descriptor()
300 	val = AFUD_READ(afu, 0x8);  in dump_afu_descriptor()
302 	val = AFUD_READ(afu, 0x10);  in dump_afu_descriptor()
304 	val = AFUD_READ(afu, 0x18);  in dump_afu_descriptor()
307 	val = AFUD_READ_CR(afu);  in dump_afu_descriptor()
311 	val = AFUD_READ_CR_OFF(afu);  in dump_afu_descriptor()
314 	val = AFUD_READ_PPPSA(afu);  in dump_afu_descriptor()
318 	val = AFUD_READ_PPPSA_OFF(afu);  in dump_afu_descriptor()
321 	val = AFUD_READ_EB(afu);  in dump_afu_descriptor()
325 	val = AFUD_READ_EB_OFF(afu);  in dump_afu_descriptor()
362 static int init_implementation_afu_regs(struct cxl_afu *afu)  in init_implementation_afu_regs()  argument
365 	cxl_p1n_write(afu, CXL_PSL_APCALLOC_A, 0xFFFFFFFEFEFEFEFEULL);  in init_implementation_afu_regs()
367 	cxl_p1n_write(afu, CXL_PSL_COALLOC_A, 0xFF000000FEFEFEFEULL);  in init_implementation_afu_regs()
369 	cxl_p1n_write(afu, CXL_PSL_SLICE_TRACE, 0x0000FFFF00000000ULL);  in init_implementation_afu_regs()
370 	cxl_p1n_write(afu, CXL_PSL_RXCTL_A, CXL_PSL_RXCTL_AFUHP_4S);  in init_implementation_afu_regs()
500 static int cxl_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in cxl_map_slice_regs()  argument
506 	p1n_base = p1_base(dev) + 0x10000 + (afu->slice * p1n_size);  in cxl_map_slice_regs()
507 	p2n_base = p2_base(dev) + (afu->slice * p2n_size);  in cxl_map_slice_regs()
508 	afu->psn_phys = p2_base(dev) + (adapter->ps_off + (afu->slice * adapter->ps_size));  in cxl_map_slice_regs()
509 	afu_desc = p2_base(dev) + adapter->afu_desc_off + (afu->slice * adapter->afu_desc_size);  in cxl_map_slice_regs()
511 	if (!(afu->p1n_mmio = ioremap(p1n_base, p1n_size)))  in cxl_map_slice_regs()
513 	if (!(afu->p2n_mmio = ioremap(p2n_base, p2n_size)))  in cxl_map_slice_regs()
516 		if (!(afu->afu_desc_mmio = ioremap(afu_desc, adapter->afu_desc_size)))  in cxl_map_slice_regs()
522 	iounmap(afu->p2n_mmio);  in cxl_map_slice_regs()
524 	iounmap(afu->p1n_mmio);  in cxl_map_slice_regs()
526 	dev_err(&afu->dev, "Error mapping AFU MMIO regions\n");  in cxl_map_slice_regs()
530 static void cxl_unmap_slice_regs(struct cxl_afu *afu)  in cxl_unmap_slice_regs()  argument
532 	if (afu->p1n_mmio)  in cxl_unmap_slice_regs()
533 		iounmap(afu->p2n_mmio);  in cxl_unmap_slice_regs()
534 	if (afu->p1n_mmio)  in cxl_unmap_slice_regs()
535 		iounmap(afu->p1n_mmio);  in cxl_unmap_slice_regs()
540 	struct cxl_afu *afu = to_cxl_afu(dev);  in cxl_release_afu()  local
544 	kfree(afu);  in cxl_release_afu()
549 	struct cxl_afu *afu;  in cxl_alloc_afu()  local
551 	if (!(afu = kzalloc(sizeof(struct cxl_afu), GFP_KERNEL)))  in cxl_alloc_afu()
554 	afu->adapter = adapter;  in cxl_alloc_afu()
555 	afu->dev.parent = &adapter->dev;  in cxl_alloc_afu()
556 	afu->dev.release = cxl_release_afu;  in cxl_alloc_afu()
557 	afu->slice = slice;  in cxl_alloc_afu()
558 	idr_init(&afu->contexts_idr);  in cxl_alloc_afu()
559 	mutex_init(&afu->contexts_lock);  in cxl_alloc_afu()
560 	spin_lock_init(&afu->afu_cntl_lock);  in cxl_alloc_afu()
561 	mutex_init(&afu->spa_mutex);  in cxl_alloc_afu()
563 	afu->prefault_mode = CXL_PREFAULT_NONE;  in cxl_alloc_afu()
564 	afu->irqs_max = afu->adapter->user_irqs;  in cxl_alloc_afu()
566 	return afu;  in cxl_alloc_afu()
570 static int cxl_read_afu_descriptor(struct cxl_afu *afu)  in cxl_read_afu_descriptor()  argument
574 	val = AFUD_READ_INFO(afu);  in cxl_read_afu_descriptor()
575 	afu->pp_irqs = AFUD_NUM_INTS_PER_PROC(val);  in cxl_read_afu_descriptor()
576 	afu->max_procs_virtualised = AFUD_NUM_PROCS(val);  in cxl_read_afu_descriptor()
577 	afu->crs_num = AFUD_NUM_CRS(val);  in cxl_read_afu_descriptor()
580 		afu->modes_supported |= CXL_MODE_DIRECTED;  in cxl_read_afu_descriptor()
582 		afu->modes_supported |= CXL_MODE_DEDICATED;  in cxl_read_afu_descriptor()
584 		afu->modes_supported |= CXL_MODE_TIME_SLICED;  in cxl_read_afu_descriptor()
586 	val = AFUD_READ_PPPSA(afu);  in cxl_read_afu_descriptor()
587 	afu->pp_size = AFUD_PPPSA_LEN(val) * 4096;  in cxl_read_afu_descriptor()
588 	afu->psa = AFUD_PPPSA_PSA(val);  in cxl_read_afu_descriptor()
589 	if ((afu->pp_psa = AFUD_PPPSA_PP(val)))  in cxl_read_afu_descriptor()
590 		afu->pp_offset = AFUD_READ_PPPSA_OFF(afu);  in cxl_read_afu_descriptor()
592 	val = AFUD_READ_CR(afu);  in cxl_read_afu_descriptor()
593 	afu->crs_len = AFUD_CR_LEN(val) * 256;  in cxl_read_afu_descriptor()
594 	afu->crs_offset = AFUD_READ_CR_OFF(afu);  in cxl_read_afu_descriptor()
599 static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu)  in cxl_afu_descriptor_looks_ok()  argument
603 	if (afu->psa && afu->adapter->ps_size <  in cxl_afu_descriptor_looks_ok()
604 			(afu->pp_offset + afu->pp_size*afu->max_procs_virtualised)) {  in cxl_afu_descriptor_looks_ok()
605 		dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n");  in cxl_afu_descriptor_looks_ok()
609 	if (afu->pp_psa && (afu->pp_size < PAGE_SIZE))  in cxl_afu_descriptor_looks_ok()
610 		dev_warn(&afu->dev, "AFU uses < PAGE_SIZE per-process PSA!");  in cxl_afu_descriptor_looks_ok()
612 	for (i = 0; i < afu->crs_num; i++) {  in cxl_afu_descriptor_looks_ok()
613 		if ((cxl_afu_cr_read32(afu, i, 0) == 0)) {  in cxl_afu_descriptor_looks_ok()
614 			dev_err(&afu->dev, "ABORTING: AFU configuration record %i is invalid\n", i);  in cxl_afu_descriptor_looks_ok()
622 static int sanitise_afu_regs(struct cxl_afu *afu)  in sanitise_afu_regs()  argument
631 	reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);  in sanitise_afu_regs()
633 		dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#.16llx\n", reg);  in sanitise_afu_regs()
634 		if (cxl_afu_reset(afu))  in sanitise_afu_regs()
636 		if (cxl_afu_disable(afu))  in sanitise_afu_regs()
638 		if (cxl_psl_purge(afu))  in sanitise_afu_regs()
641 	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);  in sanitise_afu_regs()
642 	cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, 0x0000000000000000);  in sanitise_afu_regs()
643 	cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 0x0000000000000000);  in sanitise_afu_regs()
644 	cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);  in sanitise_afu_regs()
645 	cxl_p1n_write(afu, CXL_PSL_SPOffset_An, 0x0000000000000000);  in sanitise_afu_regs()
646 	cxl_p1n_write(afu, CXL_HAURP_An, 0x0000000000000000);  in sanitise_afu_regs()
647 	cxl_p2n_write(afu, CXL_CSRP_An, 0x0000000000000000);  in sanitise_afu_regs()
648 	cxl_p2n_write(afu, CXL_AURP1_An, 0x0000000000000000);  in sanitise_afu_regs()
649 	cxl_p2n_write(afu, CXL_AURP0_An, 0x0000000000000000);  in sanitise_afu_regs()
650 	cxl_p2n_write(afu, CXL_SSTP1_An, 0x0000000000000000);  in sanitise_afu_regs()
651 	cxl_p2n_write(afu, CXL_SSTP0_An, 0x0000000000000000);  in sanitise_afu_regs()
652 	reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);  in sanitise_afu_regs()
654 		dev_warn(&afu->dev, "AFU had pending DSISR: %#.16llx\n", reg);  in sanitise_afu_regs()
656 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);  in sanitise_afu_regs()
658 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);  in sanitise_afu_regs()
660 	reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);  in sanitise_afu_regs()
663 			dev_warn(&afu->dev, "AFU had pending SERR: %#.16llx\n", reg);  in sanitise_afu_regs()
664 		cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);  in sanitise_afu_regs()
666 	reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);  in sanitise_afu_regs()
668 		dev_warn(&afu->dev, "AFU had pending error status: %#.16llx\n", reg);  in sanitise_afu_regs()
669 		cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);  in sanitise_afu_regs()
677 	struct cxl_afu *afu;  in cxl_init_afu()  local
681 	if (!(afu = cxl_alloc_afu(adapter, slice)))  in cxl_init_afu()
684 	if ((rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice)))  in cxl_init_afu()
687 	if ((rc = cxl_map_slice_regs(afu, adapter, dev)))  in cxl_init_afu()
690 	if ((rc = sanitise_afu_regs(afu)))  in cxl_init_afu()
694 	if ((rc = cxl_afu_reset(afu)))  in cxl_init_afu()
698 		dump_afu_descriptor(afu);  in cxl_init_afu()
700 	if ((rc = cxl_read_afu_descriptor(afu)))  in cxl_init_afu()
703 	if ((rc = cxl_afu_descriptor_looks_ok(afu)))  in cxl_init_afu()
706 	if ((rc = init_implementation_afu_regs(afu)))  in cxl_init_afu()
709 	if ((rc = cxl_register_serr_irq(afu)))  in cxl_init_afu()
712 	if ((rc = cxl_register_psl_irq(afu)))  in cxl_init_afu()
716 	cxl_debugfs_afu_add(afu);  in cxl_init_afu()
722 	if ((rc = cxl_register_afu(afu)))  in cxl_init_afu()
725 	if ((rc = cxl_sysfs_afu_add(afu)))  in cxl_init_afu()
729 	if ((rc = cxl_afu_select_best_mode(afu)))  in cxl_init_afu()
732 	adapter->afu[afu->slice] = afu;  in cxl_init_afu()
737 	cxl_sysfs_afu_remove(afu);  in cxl_init_afu()
739 	device_unregister(&afu->dev);  in cxl_init_afu()
741 	cxl_debugfs_afu_remove(afu);  in cxl_init_afu()
742 	cxl_release_psl_irq(afu);  in cxl_init_afu()
744 	cxl_release_serr_irq(afu);  in cxl_init_afu()
746 	cxl_unmap_slice_regs(afu);  in cxl_init_afu()
749 		kfree(afu);  in cxl_init_afu()
753 static void cxl_remove_afu(struct cxl_afu *afu)  in cxl_remove_afu()  argument
757 	if (!afu)  in cxl_remove_afu()
760 	cxl_sysfs_afu_remove(afu);  in cxl_remove_afu()
761 	cxl_debugfs_afu_remove(afu);  in cxl_remove_afu()
763 	spin_lock(&afu->adapter->afu_list_lock);  in cxl_remove_afu()
764 	afu->adapter->afu[afu->slice] = NULL;  in cxl_remove_afu()
765 	spin_unlock(&afu->adapter->afu_list_lock);  in cxl_remove_afu()
767 	cxl_context_detach_all(afu);  in cxl_remove_afu()
768 	cxl_afu_deactivate_mode(afu);  in cxl_remove_afu()
770 	cxl_release_psl_irq(afu);  in cxl_remove_afu()
771 	cxl_release_serr_irq(afu);  in cxl_remove_afu()
772 	cxl_unmap_slice_regs(afu);  in cxl_remove_afu()
774 	device_unregister(&afu->dev);  in cxl_remove_afu()
1074 	int afu;  in cxl_remove()  local
1082 	for (afu = 0; afu < adapter->slices; afu++)  in cxl_remove()
1083 		cxl_remove_afu(adapter->afu[afu]);  in cxl_remove()