356 				   struct amdgpu_ib *ib)  in sdma_v3_0_ring_emit_ib()  argument
358 	u32 vmid = (ib->vm ? ib->vm->ids[ring->idx].id : 0) & 0xf;  in sdma_v3_0_ring_emit_ib()
378 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);  in sdma_v3_0_ring_emit_ib()
379 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));  in sdma_v3_0_ring_emit_ib()
380 	amdgpu_ring_write(ring, ib->length_dw);  in sdma_v3_0_ring_emit_ib()
834 	struct amdgpu_ib ib;  in sdma_v3_0_ring_test_ib()  local
851 	memset(&ib, 0, sizeof(ib));  in sdma_v3_0_ring_test_ib()
852 	r = amdgpu_ib_get(ring, NULL, 256, &ib);  in sdma_v3_0_ring_test_ib()
858 	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |  in sdma_v3_0_ring_test_ib()
860 	ib.ptr[1] = lower_32_bits(gpu_addr);  in sdma_v3_0_ring_test_ib()
861 	ib.ptr[2] = upper_32_bits(gpu_addr);  in sdma_v3_0_ring_test_ib()
862 	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1);  in sdma_v3_0_ring_test_ib()
863 	ib.ptr[4] = 0xDEADBEEF;  in sdma_v3_0_ring_test_ib()
864 	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);  in sdma_v3_0_ring_test_ib()
865 	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);  in sdma_v3_0_ring_test_ib()
866 	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);  in sdma_v3_0_ring_test_ib()
867 	ib.length_dw = 8;  in sdma_v3_0_ring_test_ib()
869 	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,  in sdma_v3_0_ring_test_ib()
896 	amdgpu_ib_free(adev, &ib);  in sdma_v3_0_ring_test_ib()
912 static void sdma_v3_0_vm_copy_pte(struct amdgpu_ib *ib,  in sdma_v3_0_vm_copy_pte()  argument
921 		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |  in sdma_v3_0_vm_copy_pte()
923 		ib->ptr[ib->length_dw++] = bytes;  in sdma_v3_0_vm_copy_pte()
924 		ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */  in sdma_v3_0_vm_copy_pte()
925 		ib->ptr[ib->length_dw++] = lower_32_bits(src);  in sdma_v3_0_vm_copy_pte()
926 		ib->ptr[ib->length_dw++] = upper_32_bits(src);  in sdma_v3_0_vm_copy_pte()
927 		ib->ptr[ib->length_dw++] = lower_32_bits(pe);  in sdma_v3_0_vm_copy_pte()
928 		ib->ptr[ib->length_dw++] = upper_32_bits(pe);  in sdma_v3_0_vm_copy_pte()
948 static void sdma_v3_0_vm_write_pte(struct amdgpu_ib *ib,  in sdma_v3_0_vm_write_pte()  argument
962 		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |  in sdma_v3_0_vm_write_pte()
964 		ib->ptr[ib->length_dw++] = pe;  in sdma_v3_0_vm_write_pte()
965 		ib->ptr[ib->length_dw++] = upper_32_bits(pe);  in sdma_v3_0_vm_write_pte()
966 		ib->ptr[ib->length_dw++] = ndw;  in sdma_v3_0_vm_write_pte()
969 				value = amdgpu_vm_map_gart(ib->ring->adev, addr);  in sdma_v3_0_vm_write_pte()
978 			ib->ptr[ib->length_dw++] = value;  in sdma_v3_0_vm_write_pte()
979 			ib->ptr[ib->length_dw++] = upper_32_bits(value);  in sdma_v3_0_vm_write_pte()
996 static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib,  in sdma_v3_0_vm_set_pte_pde()  argument
1015 		ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_GEN_PTEPDE);  in sdma_v3_0_vm_set_pte_pde()
1016 		ib->ptr[ib->length_dw++] = pe; /* dst addr */  in sdma_v3_0_vm_set_pte_pde()
1017 		ib->ptr[ib->length_dw++] = upper_32_bits(pe);  in sdma_v3_0_vm_set_pte_pde()
1018 		ib->ptr[ib->length_dw++] = flags; /* mask */  in sdma_v3_0_vm_set_pte_pde()
1019 		ib->ptr[ib->length_dw++] = 0;  in sdma_v3_0_vm_set_pte_pde()
1020 		ib->ptr[ib->length_dw++] = value; /* value */  in sdma_v3_0_vm_set_pte_pde()
1021 		ib->ptr[ib->length_dw++] = upper_32_bits(value);  in sdma_v3_0_vm_set_pte_pde()
1022 		ib->ptr[ib->length_dw++] = incr; /* increment size */  in sdma_v3_0_vm_set_pte_pde()
1023 		ib->ptr[ib->length_dw++] = 0;  in sdma_v3_0_vm_set_pte_pde()
1024 		ib->ptr[ib->length_dw++] = ndw; /* number of entries */  in sdma_v3_0_vm_set_pte_pde()
1038 static void sdma_v3_0_vm_pad_ib(struct amdgpu_ib *ib)  in sdma_v3_0_vm_pad_ib()  argument
1040 	struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ib->ring);  in sdma_v3_0_vm_pad_ib()
1044 	pad_count = (8 - (ib->length_dw & 0x7)) % 8;  in sdma_v3_0_vm_pad_ib()
1047 			ib->ptr[ib->length_dw++] =  in sdma_v3_0_vm_pad_ib()
1051 			ib->ptr[ib->length_dw++] =  in sdma_v3_0_vm_pad_ib()
1510 static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ib *ib,  in sdma_v3_0_emit_copy_buffer()  argument
1515 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |  in sdma_v3_0_emit_copy_buffer()
1517 	ib->ptr[ib->length_dw++] = byte_count;  in sdma_v3_0_emit_copy_buffer()
1518 	ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */  in sdma_v3_0_emit_copy_buffer()
1519 	ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);  in sdma_v3_0_emit_copy_buffer()
1520 	ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);  in sdma_v3_0_emit_copy_buffer()
1521 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);  in sdma_v3_0_emit_copy_buffer()
1522 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);  in sdma_v3_0_emit_copy_buffer()
1535 static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ib *ib,  in sdma_v3_0_emit_fill_buffer()  argument
1540 	ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);  in sdma_v3_0_emit_fill_buffer()
1541 	ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);  in sdma_v3_0_emit_fill_buffer()
1542 	ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);  in sdma_v3_0_emit_fill_buffer()
1543 	ib->ptr[ib->length_dw++] = src_data;  in sdma_v3_0_emit_fill_buffer()
1544 	ib->ptr[ib->length_dw++] = byte_count;  in sdma_v3_0_emit_fill_buffer()