43 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo);
50 static void radeon_update_memory_usage(struct radeon_bo *bo,  in radeon_update_memory_usage()  argument
53 	struct radeon_device *rdev = bo->rdev;  in radeon_update_memory_usage()
54 	u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;  in radeon_update_memory_usage()
74 	struct radeon_bo *bo;  in radeon_ttm_bo_destroy()  local
76 	bo = container_of(tbo, struct radeon_bo, tbo);  in radeon_ttm_bo_destroy()
78 	radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1);  in radeon_ttm_bo_destroy()
80 	mutex_lock(&bo->rdev->gem.mutex);  in radeon_ttm_bo_destroy()
81 	list_del_init(&bo->list);  in radeon_ttm_bo_destroy()
82 	mutex_unlock(&bo->rdev->gem.mutex);  in radeon_ttm_bo_destroy()
83 	radeon_bo_clear_surface_reg(bo);  in radeon_ttm_bo_destroy()
84 	WARN_ON(!list_empty(&bo->va));  in radeon_ttm_bo_destroy()
85 	drm_gem_object_release(&bo->gem_base);  in radeon_ttm_bo_destroy()
86 	kfree(bo);  in radeon_ttm_bo_destroy()
89 bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)  in radeon_ttm_bo_is_radeon_bo()  argument
91 	if (bo->destroy == &radeon_ttm_bo_destroy)  in radeon_ttm_bo_is_radeon_bo()
184 	struct radeon_bo *bo;  in radeon_bo_create()  local
204 	bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);  in radeon_bo_create()
205 	if (bo == NULL)  in radeon_bo_create()
207 	r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size);  in radeon_bo_create()
209 		kfree(bo);  in radeon_bo_create()
212 	bo->rdev = rdev;  in radeon_bo_create()
213 	bo->surface_reg = -1;  in radeon_bo_create()
214 	INIT_LIST_HEAD(&bo->list);  in radeon_bo_create()
215 	INIT_LIST_HEAD(&bo->va);  in radeon_bo_create()
216 	bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM |  in radeon_bo_create()
220 	bo->flags = flags;  in radeon_bo_create()
223 		bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
229 		bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
235 	bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
245 	if (bo->flags & RADEON_GEM_GTT_WC)  in radeon_bo_create()
248 	bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);  in radeon_bo_create()
254 		bo->flags &= ~RADEON_GEM_GTT_WC;  in radeon_bo_create()
257 	radeon_ttm_placement_from_domain(bo, domain);  in radeon_bo_create()
260 	r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,  in radeon_bo_create()
261 			&bo->placement, page_align, !kernel, NULL,  in radeon_bo_create()
267 	*bo_ptr = bo;  in radeon_bo_create()
269 	trace_radeon_bo_create(bo);  in radeon_bo_create()
274 int radeon_bo_kmap(struct radeon_bo *bo, void **ptr)  in radeon_bo_kmap()  argument
279 	if (bo->kptr) {  in radeon_bo_kmap()
281 			*ptr = bo->kptr;  in radeon_bo_kmap()
285 	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);  in radeon_bo_kmap()
289 	bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);  in radeon_bo_kmap()
291 		*ptr = bo->kptr;  in radeon_bo_kmap()
293 	radeon_bo_check_tiling(bo, 0, 0);  in radeon_bo_kmap()
297 void radeon_bo_kunmap(struct radeon_bo *bo)  in radeon_bo_kunmap()  argument
299 	if (bo->kptr == NULL)  in radeon_bo_kunmap()
301 	bo->kptr = NULL;  in radeon_bo_kunmap()
302 	radeon_bo_check_tiling(bo, 0, 0);  in radeon_bo_kunmap()
303 	ttm_bo_kunmap(&bo->kmap);  in radeon_bo_kunmap()
306 struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo)  in radeon_bo_ref()  argument
308 	if (bo == NULL)  in radeon_bo_ref()
311 	ttm_bo_reference(&bo->tbo);  in radeon_bo_ref()
312 	return bo;  in radeon_bo_ref()
315 void radeon_bo_unref(struct radeon_bo **bo)  in radeon_bo_unref()  argument
320 	if ((*bo) == NULL)  in radeon_bo_unref()
322 	rdev = (*bo)->rdev;  in radeon_bo_unref()
323 	tbo = &((*bo)->tbo);  in radeon_bo_unref()
326 		*bo = NULL;  in radeon_bo_unref()
329 int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,  in radeon_bo_pin_restricted()  argument
334 	if (radeon_ttm_tt_has_userptr(bo->tbo.ttm))  in radeon_bo_pin_restricted()
337 	if (bo->pin_count) {  in radeon_bo_pin_restricted()
338 		bo->pin_count++;  in radeon_bo_pin_restricted()
340 			*gpu_addr = radeon_bo_gpu_offset(bo);  in radeon_bo_pin_restricted()
346 				domain_start = bo->rdev->mc.vram_start;  in radeon_bo_pin_restricted()
348 				domain_start = bo->rdev->mc.gtt_start;  in radeon_bo_pin_restricted()
350 				     (radeon_bo_gpu_offset(bo) - domain_start));  in radeon_bo_pin_restricted()
355 	radeon_ttm_placement_from_domain(bo, domain);  in radeon_bo_pin_restricted()
356 	for (i = 0; i < bo->placement.num_placement; i++) {  in radeon_bo_pin_restricted()
358 		if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&  in radeon_bo_pin_restricted()
359 		    !(bo->flags & RADEON_GEM_NO_CPU_ACCESS) &&  in radeon_bo_pin_restricted()
360 		    (!max_offset || max_offset > bo->rdev->mc.visible_vram_size))  in radeon_bo_pin_restricted()
361 			bo->placements[i].lpfn =  in radeon_bo_pin_restricted()
362 				bo->rdev->mc.visible_vram_size >> PAGE_SHIFT;  in radeon_bo_pin_restricted()
364 			bo->placements[i].lpfn = max_offset >> PAGE_SHIFT;  in radeon_bo_pin_restricted()
366 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;  in radeon_bo_pin_restricted()
369 	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);  in radeon_bo_pin_restricted()
371 		bo->pin_count = 1;  in radeon_bo_pin_restricted()
373 			*gpu_addr = radeon_bo_gpu_offset(bo);  in radeon_bo_pin_restricted()
375 			bo->rdev->vram_pin_size += radeon_bo_size(bo);  in radeon_bo_pin_restricted()
377 			bo->rdev->gart_pin_size += radeon_bo_size(bo);  in radeon_bo_pin_restricted()
379 		dev_err(bo->rdev->dev, "%p pin failed\n", bo);  in radeon_bo_pin_restricted()
384 int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)  in radeon_bo_pin()  argument
386 	return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr);  in radeon_bo_pin()
389 int radeon_bo_unpin(struct radeon_bo *bo)  in radeon_bo_unpin()  argument
393 	if (!bo->pin_count) {  in radeon_bo_unpin()
394 		dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo);  in radeon_bo_unpin()
397 	bo->pin_count--;  in radeon_bo_unpin()
398 	if (bo->pin_count)  in radeon_bo_unpin()
400 	for (i = 0; i < bo->placement.num_placement; i++) {  in radeon_bo_unpin()
401 		bo->placements[i].lpfn = 0;  in radeon_bo_unpin()
402 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;  in radeon_bo_unpin()
404 	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);  in radeon_bo_unpin()
406 		if (bo->tbo.mem.mem_type == TTM_PL_VRAM)  in radeon_bo_unpin()
407 			bo->rdev->vram_pin_size -= radeon_bo_size(bo);  in radeon_bo_unpin()
409 			bo->rdev->gart_pin_size -= radeon_bo_size(bo);  in radeon_bo_unpin()
411 		dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo);  in radeon_bo_unpin()
429 	struct radeon_bo *bo, *n;  in radeon_bo_force_delete()  local
435 	list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {  in radeon_bo_force_delete()
437 			&bo->gem_base, bo, (unsigned long)bo->gem_base.size,  in radeon_bo_force_delete()
438 			*((unsigned long *)&bo->gem_base.refcount));  in radeon_bo_force_delete()
439 		mutex_lock(&bo->rdev->gem.mutex);  in radeon_bo_force_delete()
440 		list_del_init(&bo->list);  in radeon_bo_force_delete()
441 		mutex_unlock(&bo->rdev->gem.mutex);  in radeon_bo_force_delete()
443 		drm_gem_object_unreference_unlocked(&bo->gem_base);  in radeon_bo_force_delete()
537 		struct radeon_bo *bo = lobj->robj;  in radeon_bo_list_validate()  local
538 		if (!bo->pin_count) {  in radeon_bo_list_validate()
542 				radeon_mem_type_to_domain(bo->tbo.mem.mem_type);  in radeon_bo_list_validate()
560 			radeon_ttm_placement_from_domain(bo, domain);  in radeon_bo_list_validate()
562 				radeon_uvd_force_into_uvd_segment(bo, allowed);  in radeon_bo_list_validate()
565 			r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);  in radeon_bo_list_validate()
579 		lobj->gpu_offset = radeon_bo_gpu_offset(bo);  in radeon_bo_list_validate()
580 		lobj->tiling_flags = bo->tiling_flags;  in radeon_bo_list_validate()
591 int radeon_bo_get_surface_reg(struct radeon_bo *bo)  in radeon_bo_get_surface_reg()  argument
593 	struct radeon_device *rdev = bo->rdev;  in radeon_bo_get_surface_reg()
599 	lockdep_assert_held(&bo->tbo.resv->lock.base);  in radeon_bo_get_surface_reg()
601 	if (!bo->tiling_flags)  in radeon_bo_get_surface_reg()
604 	if (bo->surface_reg >= 0) {  in radeon_bo_get_surface_reg()
605 		reg = &rdev->surface_regs[bo->surface_reg];  in radeon_bo_get_surface_reg()
606 		i = bo->surface_reg;  in radeon_bo_get_surface_reg()
614 		if (!reg->bo)  in radeon_bo_get_surface_reg()
617 		old_object = reg->bo;  in radeon_bo_get_surface_reg()
628 		old_object = reg->bo;  in radeon_bo_get_surface_reg()
636 	bo->surface_reg = i;  in radeon_bo_get_surface_reg()
637 	reg->bo = bo;  in radeon_bo_get_surface_reg()
640 	radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,  in radeon_bo_get_surface_reg()
641 			       bo->tbo.mem.start << PAGE_SHIFT,  in radeon_bo_get_surface_reg()
642 			       bo->tbo.num_pages << PAGE_SHIFT);  in radeon_bo_get_surface_reg()
646 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo)  in radeon_bo_clear_surface_reg()  argument
648 	struct radeon_device *rdev = bo->rdev;  in radeon_bo_clear_surface_reg()
651 	if (bo->surface_reg == -1)  in radeon_bo_clear_surface_reg()
654 	reg = &rdev->surface_regs[bo->surface_reg];  in radeon_bo_clear_surface_reg()
655 	radeon_clear_surface_reg(rdev, bo->surface_reg);  in radeon_bo_clear_surface_reg()
657 	reg->bo = NULL;  in radeon_bo_clear_surface_reg()
658 	bo->surface_reg = -1;  in radeon_bo_clear_surface_reg()
661 int radeon_bo_set_tiling_flags(struct radeon_bo *bo,  in radeon_bo_set_tiling_flags()  argument
664 	struct radeon_device *rdev = bo->rdev;  in radeon_bo_set_tiling_flags()
712 	r = radeon_bo_reserve(bo, false);  in radeon_bo_set_tiling_flags()
715 	bo->tiling_flags = tiling_flags;  in radeon_bo_set_tiling_flags()
716 	bo->pitch = pitch;  in radeon_bo_set_tiling_flags()
717 	radeon_bo_unreserve(bo);  in radeon_bo_set_tiling_flags()
721 void radeon_bo_get_tiling_flags(struct radeon_bo *bo,  in radeon_bo_get_tiling_flags()  argument
725 	lockdep_assert_held(&bo->tbo.resv->lock.base);  in radeon_bo_get_tiling_flags()
728 		*tiling_flags = bo->tiling_flags;  in radeon_bo_get_tiling_flags()
730 		*pitch = bo->pitch;  in radeon_bo_get_tiling_flags()
733 int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,  in radeon_bo_check_tiling()  argument
737 		lockdep_assert_held(&bo->tbo.resv->lock.base);  in radeon_bo_check_tiling()
739 	if (!(bo->tiling_flags & RADEON_TILING_SURFACE))  in radeon_bo_check_tiling()
743 		radeon_bo_clear_surface_reg(bo);  in radeon_bo_check_tiling()
747 	if (bo->tbo.mem.mem_type != TTM_PL_VRAM) {  in radeon_bo_check_tiling()
751 		if (bo->surface_reg >= 0)  in radeon_bo_check_tiling()
752 			radeon_bo_clear_surface_reg(bo);  in radeon_bo_check_tiling()
756 	if ((bo->surface_reg >= 0) && !has_moved)  in radeon_bo_check_tiling()
759 	return radeon_bo_get_surface_reg(bo);  in radeon_bo_check_tiling()
762 void radeon_bo_move_notify(struct ttm_buffer_object *bo,  in radeon_bo_move_notify()  argument
767 	if (!radeon_ttm_bo_is_radeon_bo(bo))  in radeon_bo_move_notify()
770 	rbo = container_of(bo, struct radeon_bo, tbo);  in radeon_bo_move_notify()
778 	radeon_update_memory_usage(rbo, bo->mem.mem_type, -1);  in radeon_bo_move_notify()
782 int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)  in radeon_bo_fault_reserve_notify()  argument
789 	if (!radeon_ttm_bo_is_radeon_bo(bo))  in radeon_bo_fault_reserve_notify()
791 	rbo = container_of(bo, struct radeon_bo, tbo);  in radeon_bo_fault_reserve_notify()
794 	if (bo->mem.mem_type != TTM_PL_VRAM)  in radeon_bo_fault_reserve_notify()
797 	size = bo->mem.num_pages << PAGE_SHIFT;  in radeon_bo_fault_reserve_notify()
798 	offset = bo->mem.start << PAGE_SHIFT;  in radeon_bo_fault_reserve_notify()
811 	r = ttm_bo_validate(bo, &rbo->placement, false, false);  in radeon_bo_fault_reserve_notify()
814 		return ttm_bo_validate(bo, &rbo->placement, false, false);  in radeon_bo_fault_reserve_notify()
819 	offset = bo->mem.start << PAGE_SHIFT;  in radeon_bo_fault_reserve_notify()
827 int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)  in radeon_bo_wait()  argument
831 	r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, NULL);  in radeon_bo_wait()
835 		*mem_type = bo->tbo.mem.mem_type;  in radeon_bo_wait()
837 	r = ttm_bo_wait(&bo->tbo, true, true, no_wait);  in radeon_bo_wait()
838 	ttm_bo_unreserve(&bo->tbo);  in radeon_bo_wait()
850 void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence,  in radeon_bo_fence()  argument
853 	struct reservation_object *resv = bo->tbo.resv;  in radeon_bo_fence()