Lines Matching refs:afu
23 static int afu_control(struct cxl_afu *afu, u64 command, in afu_control() argument
26 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in afu_control()
30 spin_lock(&afu->afu_cntl_lock); in afu_control()
33 trace_cxl_afu_ctrl(afu, command); in afu_control()
35 cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command); in afu_control()
37 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in afu_control()
40 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n"); in afu_control()
47 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in afu_control()
50 afu->enabled = enabled; in afu_control()
52 trace_cxl_afu_ctrl_done(afu, command, rc); in afu_control()
53 spin_unlock(&afu->afu_cntl_lock); in afu_control()
58 static int afu_enable(struct cxl_afu *afu) in afu_enable() argument
62 return afu_control(afu, CXL_AFU_Cntl_An_E, in afu_enable()
67 int cxl_afu_disable(struct cxl_afu *afu) in cxl_afu_disable() argument
71 return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled, in cxl_afu_disable()
76 int cxl_afu_reset(struct cxl_afu *afu) in cxl_afu_reset() argument
80 return afu_control(afu, CXL_AFU_Cntl_An_RA, in cxl_afu_reset()
86 static int afu_check_and_enable(struct cxl_afu *afu) in afu_check_and_enable() argument
88 if (afu->enabled) in afu_check_and_enable()
90 return afu_enable(afu); in afu_check_and_enable()
93 int cxl_psl_purge(struct cxl_afu *afu) in cxl_psl_purge() argument
95 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); in cxl_psl_purge()
96 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in cxl_psl_purge()
102 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc); in cxl_psl_purge()
108 cxl_afu_disable(afu); in cxl_psl_purge()
111 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, in cxl_psl_purge()
114 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); in cxl_psl_purge()
118 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n"); in cxl_psl_purge()
122 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); in cxl_psl_purge()
125 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); in cxl_psl_purge()
126 …dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%.16llx, DAR: 0x%.16llx… in cxl_psl_purge()
127 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); in cxl_psl_purge()
129 …dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%.16llx\n",… in cxl_psl_purge()
130 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); in cxl_psl_purge()
134 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); in cxl_psl_purge()
139 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, in cxl_psl_purge()
142 trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc); in cxl_psl_purge()
164 static int alloc_spa(struct cxl_afu *afu) in alloc_spa() argument
169 afu->spa_order = 0; in alloc_spa()
171 afu->spa_order++; in alloc_spa()
172 afu->spa_size = (1 << afu->spa_order) * PAGE_SIZE; in alloc_spa()
173 afu->spa_max_procs = spa_max_procs(afu->spa_size); in alloc_spa()
174 } while (afu->spa_max_procs < afu->num_procs); in alloc_spa()
176 WARN_ON(afu->spa_size > 0x100000); /* Max size supported by the hardware */ in alloc_spa()
178 if (!(afu->spa = (struct cxl_process_element *) in alloc_spa()
179 __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->spa_order))) { in alloc_spa()
184 1<<afu->spa_order, afu->spa_max_procs, afu->num_procs); in alloc_spa()
186 afu->sw_command_status = (__be64 *)((char *)afu->spa + in alloc_spa()
187 ((afu->spa_max_procs + 3) * 128)); in alloc_spa()
189 spap = virt_to_phys(afu->spa) & CXL_PSL_SPAP_Addr; in alloc_spa()
190 spap |= ((afu->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size; in alloc_spa()
192 …: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", afu->spa, afu->spa_max_procs, afu->sw_… in alloc_spa()
193 cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap); in alloc_spa()
198 static void release_spa(struct cxl_afu *afu) in release_spa() argument
200 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); in release_spa()
201 free_pages((unsigned long) afu->spa, afu->spa_order); in release_spa()
232 int cxl_afu_slbia(struct cxl_afu *afu) in cxl_afu_slbia() argument
237 cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL); in cxl_afu_slbia()
238 while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) { in cxl_afu_slbia()
240 dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n"); in cxl_afu_slbia()
248 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1) in cxl_write_sstp() argument
253 cxl_p2n_write(afu, CXL_SSTP1_An, 0); in cxl_write_sstp()
256 if ((rc = cxl_afu_slbia(afu))) in cxl_write_sstp()
260 cxl_p2n_write(afu, CXL_SSTP0_An, sstp0); in cxl_write_sstp()
263 cxl_p2n_write(afu, CXL_SSTP1_An, sstp1); in cxl_write_sstp()
271 struct cxl *adapter = ctx->afu->adapter; in slb_invalid()
274 WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex)); in slb_invalid()
298 WARN_ON(!ctx->afu->enabled); in do_process_element_cmd()
302 *(ctx->afu->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe); in do_process_element_cmd()
304 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe); in do_process_element_cmd()
307 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n"); in do_process_element_cmd()
311 state = be64_to_cpup(ctx->afu->sw_command_status); in do_process_element_cmd()
339 mutex_lock(&ctx->afu->spa_mutex); in add_process_element()
344 mutex_unlock(&ctx->afu->spa_mutex); in add_process_element()
356 mutex_lock(&ctx->afu->spa_mutex); in terminate_process_element()
362 mutex_unlock(&ctx->afu->spa_mutex); in terminate_process_element()
370 mutex_lock(&ctx->afu->spa_mutex); in remove_process_element()
376 mutex_unlock(&ctx->afu->spa_mutex); in remove_process_element()
384 if (!ctx->afu->pp_size || ctx->master) { in assign_psn_space()
385 ctx->psn_phys = ctx->afu->psn_phys; in assign_psn_space()
386 ctx->psn_size = ctx->afu->adapter->ps_size; in assign_psn_space()
388 ctx->psn_phys = ctx->afu->psn_phys + in assign_psn_space()
389 (ctx->afu->pp_offset + ctx->afu->pp_size * ctx->pe); in assign_psn_space()
390 ctx->psn_size = ctx->afu->pp_size; in assign_psn_space()
394 static int activate_afu_directed(struct cxl_afu *afu) in activate_afu_directed() argument
398 dev_info(&afu->dev, "Activating AFU directed mode\n"); in activate_afu_directed()
400 if (alloc_spa(afu)) in activate_afu_directed()
403 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU); in activate_afu_directed()
404 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); in activate_afu_directed()
405 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); in activate_afu_directed()
407 afu->current_mode = CXL_MODE_DIRECTED; in activate_afu_directed()
408 afu->num_procs = afu->max_procs_virtualised; in activate_afu_directed()
410 if ((rc = cxl_chardev_m_afu_add(afu))) in activate_afu_directed()
413 if ((rc = cxl_sysfs_afu_m_add(afu))) in activate_afu_directed()
416 if ((rc = cxl_chardev_s_afu_add(afu))) in activate_afu_directed()
421 cxl_sysfs_afu_m_remove(afu); in activate_afu_directed()
423 cxl_chardev_afu_remove(afu); in activate_afu_directed()
480 if ((result = afu_check_and_enable(ctx->afu))) in attach_afu_directed()
488 static int deactivate_afu_directed(struct cxl_afu *afu) in deactivate_afu_directed() argument
490 dev_info(&afu->dev, "Deactivating AFU directed mode\n"); in deactivate_afu_directed()
492 afu->current_mode = 0; in deactivate_afu_directed()
493 afu->num_procs = 0; in deactivate_afu_directed()
495 cxl_sysfs_afu_m_remove(afu); in deactivate_afu_directed()
496 cxl_chardev_afu_remove(afu); in deactivate_afu_directed()
498 cxl_afu_reset(afu); in deactivate_afu_directed()
499 cxl_afu_disable(afu); in deactivate_afu_directed()
500 cxl_psl_purge(afu); in deactivate_afu_directed()
502 release_spa(afu); in deactivate_afu_directed()
507 static int activate_dedicated_process(struct cxl_afu *afu) in activate_dedicated_process() argument
509 dev_info(&afu->dev, "Activating dedicated process mode\n"); in activate_dedicated_process()
511 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); in activate_dedicated_process()
513 cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */ in activate_dedicated_process()
514 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */ in activate_dedicated_process()
515 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); in activate_dedicated_process()
516 cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID)); in activate_dedicated_process()
517 cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */ in activate_dedicated_process()
518 cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1)); in activate_dedicated_process()
520 cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */ in activate_dedicated_process()
521 cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */ in activate_dedicated_process()
522 cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */ in activate_dedicated_process()
524 afu->current_mode = CXL_MODE_DEDICATED; in activate_dedicated_process()
525 afu->num_procs = 1; in activate_dedicated_process()
527 return cxl_chardev_d_afu_add(afu); in activate_dedicated_process()
532 struct cxl_afu *afu = ctx->afu; in attach_dedicated() local
545 cxl_p2n_write(afu, CXL_PSL_PID_TID_An, (u64)current->pid << 32); in attach_dedicated()
546 cxl_p1n_write(afu, CXL_PSL_SR_An, sr); in attach_dedicated()
548 if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1))) in attach_dedicated()
553 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, in attach_dedicated()
558 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64) in attach_dedicated()
564 cxl_p2n_write(afu, CXL_PSL_AMR_An, amr); in attach_dedicated()
569 if ((rc = cxl_afu_reset(afu))) in attach_dedicated()
572 cxl_p2n_write(afu, CXL_PSL_WED_An, wed); in attach_dedicated()
574 return afu_enable(afu); in attach_dedicated()
577 static int deactivate_dedicated_process(struct cxl_afu *afu) in deactivate_dedicated_process() argument
579 dev_info(&afu->dev, "Deactivating dedicated process mode\n"); in deactivate_dedicated_process()
581 afu->current_mode = 0; in deactivate_dedicated_process()
582 afu->num_procs = 0; in deactivate_dedicated_process()
584 cxl_chardev_afu_remove(afu); in deactivate_dedicated_process()
589 int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode) in _cxl_afu_deactivate_mode() argument
592 return deactivate_afu_directed(afu); in _cxl_afu_deactivate_mode()
594 return deactivate_dedicated_process(afu); in _cxl_afu_deactivate_mode()
598 int cxl_afu_deactivate_mode(struct cxl_afu *afu) in cxl_afu_deactivate_mode() argument
600 return _cxl_afu_deactivate_mode(afu, afu->current_mode); in cxl_afu_deactivate_mode()
603 int cxl_afu_activate_mode(struct cxl_afu *afu, int mode) in cxl_afu_activate_mode() argument
607 if (!(mode & afu->modes_supported)) in cxl_afu_activate_mode()
611 return activate_afu_directed(afu); in cxl_afu_activate_mode()
613 return activate_dedicated_process(afu); in cxl_afu_activate_mode()
621 if (ctx->afu->current_mode == CXL_MODE_DIRECTED) in cxl_attach_process()
624 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) in cxl_attach_process()
632 cxl_afu_reset(ctx->afu); in detach_process_native_dedicated()
633 cxl_afu_disable(ctx->afu); in detach_process_native_dedicated()
634 cxl_psl_purge(ctx->afu); in detach_process_native_dedicated()
654 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) in cxl_detach_process()
660 int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info) in cxl_get_irq() argument
664 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); in cxl_get_irq()
665 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); in cxl_get_irq()
666 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An); in cxl_get_irq()
667 pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An); in cxl_get_irq()
670 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An); in cxl_get_irq()
671 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); in cxl_get_irq()
676 static void recover_psl_err(struct cxl_afu *afu, u64 errstat) in recover_psl_err() argument
683 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); in recover_psl_err()
684 cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE); in recover_psl_err()
687 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat); in recover_psl_err()
694 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc); in cxl_ack_irq()
696 recover_psl_err(ctx->afu, psl_reset_mask); in cxl_ack_irq()
701 int cxl_check_error(struct cxl_afu *afu) in cxl_check_error() argument
703 return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL); in cxl_check_error()