37 static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = {
60 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
67 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
68 		uint32_t sh_mem_config,	uint32_t sh_mem_ape1_base,
69 		uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
74 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
75 				uint32_t hpd_size, uint64_t hpd_gpu_addr);
76 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
77 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
78 			uint32_t queue_id, uint32_t __user *wptr);
81 				uint32_t pipe_id, uint32_t queue_id);
83 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
84 				unsigned int timeout, uint32_t pipe_id,
85 				uint32_t queue_id);
92 					uint32_t cntl_val,
93 					uint32_t addr_hi,
94 					uint32_t addr_lo);
96 					uint32_t gfx_index_val,
97 					uint32_t sq_cmd);
98 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
321 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)  in get_max_engine_clock_in_mhz()
334 static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)  in write_register()
341 static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)  in read_register()
348 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,  in lock_srbm()
349 			uint32_t queue, uint32_t vmid)  in lock_srbm()
352 	uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue);  in lock_srbm()
366 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,  in acquire_queue()
367 				uint32_t queue_id)  in acquire_queue()
369 	uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1;  in acquire_queue()
370 	uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);  in acquire_queue()
380 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,  in kgd_program_sh_mem_settings()
381 					uint32_t sh_mem_config,  in kgd_program_sh_mem_settings()
382 					uint32_t sh_mem_ape1_base,  in kgd_program_sh_mem_settings()
383 					uint32_t sh_mem_ape1_limit,  in kgd_program_sh_mem_settings()
384 					uint32_t sh_mem_bases)  in kgd_program_sh_mem_settings()
406 	uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |  in kgd_set_pasid_vmid_mapping()
409 	write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),  in kgd_set_pasid_vmid_mapping()
418 	write_register(kgd, IH_VMID_0_LUT + vmid * sizeof(uint32_t),  in kgd_set_pasid_vmid_mapping()
424 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,  in kgd_init_pipeline()
425 				uint32_t hpd_size, uint64_t hpd_gpu_addr)  in kgd_init_pipeline()
427 	uint32_t mec = (pipe_id / CIK_PIPE_PER_MEC) + 1;  in kgd_init_pipeline()
428 	uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC);  in kgd_init_pipeline()
442 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)  in kgd_init_interrupts()
444 	uint32_t mec;  in kgd_init_interrupts()
445 	uint32_t pipe;  in kgd_init_interrupts()
460 static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)  in get_sdma_base_addr()
462 	uint32_t retval;  in get_sdma_base_addr()
482 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,  in kgd_hqd_load()
483 			uint32_t queue_id, uint32_t __user *wptr)  in kgd_hqd_load()
485 	uint32_t wptr_shadow, is_wptr_shadow_valid;  in kgd_hqd_load()
562 	uint32_t sdma_base_addr;  in kgd_hqd_sdma_load()
599 				uint32_t pipe_id, uint32_t queue_id)  in kgd_hqd_is_occupied()
601 	uint32_t act;  in kgd_hqd_is_occupied()
603 	uint32_t low, high;  in kgd_hqd_is_occupied()
622 	uint32_t sdma_base_addr;  in kgd_hqd_sdma_is_occupied()
623 	uint32_t sdma_rlc_rb_cntl;  in kgd_hqd_sdma_is_occupied()
637 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,  in kgd_hqd_destroy()
638 				unsigned int timeout, uint32_t pipe_id,  in kgd_hqd_destroy()
639 				uint32_t queue_id)  in kgd_hqd_destroy()
641 	uint32_t temp;  in kgd_hqd_destroy()
670 	uint32_t sdma_base_addr;  in kgd_hqd_sdma_destroy()
671 	uint32_t temp;  in kgd_hqd_sdma_destroy()
722 					uint32_t cntl_val,  in kgd_address_watch_execute()
723 					uint32_t addr_hi,  in kgd_address_watch_execute()
724 					uint32_t addr_lo)  in kgd_address_watch_execute()
759 					uint32_t gfx_index_val,  in kgd_wave_control_execute()
760 					uint32_t sq_cmd)  in kgd_wave_control_execute()
763 	uint32_t data;  in kgd_wave_control_execute()
782 static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,  in kgd_address_watch_get_offset()
791 	uint32_t reg;  in get_atc_vmid_pasid_mapping_valid()
801 	uint32_t reg;  in get_atc_vmid_pasid_mapping_pasid()