44 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
48 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
50 static uint64_t get_vmem_size(struct kgd_dev *kgd);
51 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
53 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
54 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
60 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,
64 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
67 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
70 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
72 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
73 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
76 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
79 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
80 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
198 static int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,  in alloc_gtt_mem()  argument
202 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in alloc_gtt_mem()
206 	BUG_ON(kgd == NULL);  in alloc_gtt_mem()
259 static void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)  in free_gtt_mem()  argument
273 static uint64_t get_vmem_size(struct kgd_dev *kgd)  in get_vmem_size()  argument
275 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in get_vmem_size()
277 	BUG_ON(kgd == NULL);  in get_vmem_size()
282 static uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)  in get_gpu_clock_counter()  argument
284 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in get_gpu_clock_counter()
289 static uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)  in get_max_engine_clock_in_mhz()  argument
291 	struct radeon_device *rdev = (struct radeon_device *)kgd;  in get_max_engine_clock_in_mhz()
297 static inline struct radeon_device *get_radeon_device(struct kgd_dev *kgd)  in get_radeon_device()  argument
299 	return (struct radeon_device *)kgd;  in get_radeon_device()
302 static void write_register(struct kgd_dev *kgd, uint32_t offset, uint32_t value)  in write_register()  argument
304 	struct radeon_device *rdev = get_radeon_device(kgd);  in write_register()
309 static uint32_t read_register(struct kgd_dev *kgd, uint32_t offset)  in read_register()  argument
311 	struct radeon_device *rdev = get_radeon_device(kgd);  in read_register()
316 static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe,  in lock_srbm()  argument
319 	struct radeon_device *rdev = get_radeon_device(kgd);  in lock_srbm()
323 	write_register(kgd, SRBM_GFX_CNTL, value);  in lock_srbm()
326 static void unlock_srbm(struct kgd_dev *kgd)  in unlock_srbm()  argument
328 	struct radeon_device *rdev = get_radeon_device(kgd);  in unlock_srbm()
330 	write_register(kgd, SRBM_GFX_CNTL, 0);  in unlock_srbm()
334 static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,  in acquire_queue()  argument
340 	lock_srbm(kgd, mec, pipe, queue_id, 0);  in acquire_queue()
343 static void release_queue(struct kgd_dev *kgd)  in release_queue()  argument
345 	unlock_srbm(kgd);  in release_queue()
348 static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid,  in kgd_program_sh_mem_settings()  argument
354 	lock_srbm(kgd, 0, 0, 0, vmid);  in kgd_program_sh_mem_settings()
356 	write_register(kgd, SH_MEM_CONFIG, sh_mem_config);  in kgd_program_sh_mem_settings()
357 	write_register(kgd, SH_MEM_APE1_BASE, sh_mem_ape1_base);  in kgd_program_sh_mem_settings()
358 	write_register(kgd, SH_MEM_APE1_LIMIT, sh_mem_ape1_limit);  in kgd_program_sh_mem_settings()
359 	write_register(kgd, SH_MEM_BASES, sh_mem_bases);  in kgd_program_sh_mem_settings()
361 	unlock_srbm(kgd);  in kgd_program_sh_mem_settings()
364 static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,  in kgd_set_pasid_vmid_mapping()  argument
377 	write_register(kgd, ATC_VMID0_PASID_MAPPING + vmid*sizeof(uint32_t),  in kgd_set_pasid_vmid_mapping()
380 	while (!(read_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS) &  in kgd_set_pasid_vmid_mapping()
383 	write_register(kgd, ATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid);  in kgd_set_pasid_vmid_mapping()
386 	write_register(kgd, IH_VMID_0_LUT + vmid * sizeof(uint32_t),  in kgd_set_pasid_vmid_mapping()
392 static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,  in kgd_init_pipeline()  argument
398 	lock_srbm(kgd, mec, pipe, 0, 0);  in kgd_init_pipeline()
399 	write_register(kgd, CP_HPD_EOP_BASE_ADDR,  in kgd_init_pipeline()
401 	write_register(kgd, CP_HPD_EOP_BASE_ADDR_HI,  in kgd_init_pipeline()
403 	write_register(kgd, CP_HPD_EOP_VMID, 0);  in kgd_init_pipeline()
404 	write_register(kgd, CP_HPD_EOP_CONTROL, hpd_size);  in kgd_init_pipeline()
405 	unlock_srbm(kgd);  in kgd_init_pipeline()
432 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,  in kgd_hqd_load()  argument
442 	acquire_queue(kgd, pipe_id, queue_id);  in kgd_hqd_load()
443 	write_register(kgd, CP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo);  in kgd_hqd_load()
444 	write_register(kgd, CP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi);  in kgd_hqd_load()
445 	write_register(kgd, CP_MQD_CONTROL, m->cp_mqd_control);  in kgd_hqd_load()
447 	write_register(kgd, CP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo);  in kgd_hqd_load()
448 	write_register(kgd, CP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi);  in kgd_hqd_load()
449 	write_register(kgd, CP_HQD_PQ_CONTROL, m->cp_hqd_pq_control);  in kgd_hqd_load()
451 	write_register(kgd, CP_HQD_IB_CONTROL, m->cp_hqd_ib_control);  in kgd_hqd_load()
452 	write_register(kgd, CP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo);  in kgd_hqd_load()
453 	write_register(kgd, CP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi);  in kgd_hqd_load()
455 	write_register(kgd, CP_HQD_IB_RPTR, m->cp_hqd_ib_rptr);  in kgd_hqd_load()
457 	write_register(kgd, CP_HQD_PERSISTENT_STATE,  in kgd_hqd_load()
459 	write_register(kgd, CP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd);  in kgd_hqd_load()
460 	write_register(kgd, CP_HQD_MSG_TYPE, m->cp_hqd_msg_type);  in kgd_hqd_load()
462 	write_register(kgd, CP_HQD_ATOMIC0_PREOP_LO,  in kgd_hqd_load()
465 	write_register(kgd, CP_HQD_ATOMIC0_PREOP_HI,  in kgd_hqd_load()
468 	write_register(kgd, CP_HQD_ATOMIC1_PREOP_LO,  in kgd_hqd_load()
471 	write_register(kgd, CP_HQD_ATOMIC1_PREOP_HI,  in kgd_hqd_load()
474 	write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR,  in kgd_hqd_load()
477 	write_register(kgd, CP_HQD_PQ_RPTR_REPORT_ADDR_HI,  in kgd_hqd_load()
480 	write_register(kgd, CP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr);  in kgd_hqd_load()
482 	write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR,  in kgd_hqd_load()
485 	write_register(kgd, CP_HQD_PQ_WPTR_POLL_ADDR_HI,  in kgd_hqd_load()
488 	write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL,  in kgd_hqd_load()
491 	write_register(kgd, CP_HQD_VMID, m->cp_hqd_vmid);  in kgd_hqd_load()
493 	write_register(kgd, CP_HQD_QUANTUM, m->cp_hqd_quantum);  in kgd_hqd_load()
495 	write_register(kgd, CP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority);  in kgd_hqd_load()
496 	write_register(kgd, CP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority);  in kgd_hqd_load()
498 	write_register(kgd, CP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr);  in kgd_hqd_load()
501 		write_register(kgd, CP_HQD_PQ_WPTR, wptr_shadow);  in kgd_hqd_load()
503 	write_register(kgd, CP_HQD_ACTIVE, m->cp_hqd_active);  in kgd_hqd_load()
504 	release_queue(kgd);  in kgd_hqd_load()
509 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)  in kgd_hqd_sdma_load()  argument
517 	write_register(kgd,  in kgd_hqd_sdma_load()
521 	write_register(kgd,  in kgd_hqd_sdma_load()
525 	write_register(kgd,  in kgd_hqd_sdma_load()
529 	write_register(kgd,  in kgd_hqd_sdma_load()
533 	write_register(kgd,  in kgd_hqd_sdma_load()
537 	write_register(kgd,  in kgd_hqd_sdma_load()
541 	write_register(kgd,  in kgd_hqd_sdma_load()
548 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,  in kgd_hqd_is_occupied()  argument
555 	acquire_queue(kgd, pipe_id, queue_id);  in kgd_hqd_is_occupied()
556 	act = read_register(kgd, CP_HQD_ACTIVE);  in kgd_hqd_is_occupied()
561 		if (low == read_register(kgd, CP_HQD_PQ_BASE) &&  in kgd_hqd_is_occupied()
562 				high == read_register(kgd, CP_HQD_PQ_BASE_HI))  in kgd_hqd_is_occupied()
565 	release_queue(kgd);  in kgd_hqd_is_occupied()
569 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)  in kgd_hqd_sdma_is_occupied()  argument
578 	sdma_rlc_rb_cntl = read_register(kgd,  in kgd_hqd_sdma_is_occupied()
587 static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,  in kgd_hqd_destroy()  argument
593 	acquire_queue(kgd, pipe_id, queue_id);  in kgd_hqd_destroy()
594 	write_register(kgd, CP_HQD_PQ_DOORBELL_CONTROL, 0);  in kgd_hqd_destroy()
596 	write_register(kgd, CP_HQD_DEQUEUE_REQUEST, reset_type);  in kgd_hqd_destroy()
599 		temp = read_register(kgd, CP_HQD_ACTIVE);  in kgd_hqd_destroy()
605 			release_queue(kgd);  in kgd_hqd_destroy()
612 	release_queue(kgd);  in kgd_hqd_destroy()
616 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,  in kgd_hqd_sdma_destroy()  argument
626 	temp = read_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL);  in kgd_hqd_sdma_destroy()
628 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_CNTL, temp);  in kgd_hqd_sdma_destroy()
631 		temp = read_register(kgd, sdma_base_addr +  in kgd_hqd_sdma_destroy()
641 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_DOORBELL, 0);  in kgd_hqd_sdma_destroy()
642 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_RPTR, 0);  in kgd_hqd_sdma_destroy()
643 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_WPTR, 0);  in kgd_hqd_sdma_destroy()
644 	write_register(kgd, sdma_base_addr + SDMA0_RLC0_RB_BASE, 0);  in kgd_hqd_sdma_destroy()
649 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)  in get_fw_version()  argument
651 	struct radeon_device *rdev = (struct radeon_device *) kgd;  in get_fw_version()
654 	BUG_ON(kgd == NULL || rdev->mec_fw == NULL);  in get_fw_version()