/* * SPDX-License-Identifier: MIT * * Copyright © 2014-2016 Intel Corporation */ #include #include "gt/intel_gt.h" #include "gt/intel_tlb.h" #include "i915_drv.h" #include "i915_gem_object.h" #include "i915_scatterlist.h" #include "i915_gem_lmem.h" #include "i915_gem_mman.h" void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { struct drm_i915_private *i915 = to_i915(obj->base.dev); unsigned long supported = RUNTIME_INFO(i915)->page_sizes; bool shrinkable; int i; assert_object_held_shared(obj); if (i915_gem_object_is_volatile(obj)) obj->mm.madv = I915_MADV_DONTNEED; /* Make the pages coherent with the GPU (flushing any swapin). */ if (obj->cache_dirty) { WARN_ON_ONCE(IS_DGFX(i915)); obj->write_domain = 0; if (i915_gem_object_has_struct_page(obj)) drm_clflush_sg(pages); obj->cache_dirty = false; } obj->mm.get_page.sg_pos = pages->sgl; obj->mm.get_page.sg_idx = 0; obj->mm.get_dma_page.sg_pos = pages->sgl; obj->mm.get_dma_page.sg_idx = 0; obj->mm.pages = pages; obj->mm.page_sizes.phys = i915_sg_dma_sizes(pages->sgl); GEM_BUG_ON(!obj->mm.page_sizes.phys); /* * Calculate the supported page-sizes which fit into the given * sg_page_sizes. This will give us the page-sizes which we may be able * to use opportunistically when later inserting into the GTT. For * example if phys=2G, then in theory we should be able to use 1G, 2M, * 64K or 4K pages, although in practice this will depend on a number of * other factors. */ obj->mm.page_sizes.sg = 0; for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { if (obj->mm.page_sizes.phys & ~0u << i) obj->mm.page_sizes.sg |= BIT(i); } GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg)); shrinkable = i915_gem_object_is_shrinkable(obj); if (i915_gem_object_is_tiled(obj) && i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) { GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); i915_gem_object_set_tiling_quirk(obj); GEM_BUG_ON(!list_empty(&obj->mm.link)); atomic_inc(&obj->mm.shrink_pin); shrinkable = false; } if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) { struct list_head *list; unsigned long flags; assert_object_held(obj); spin_lock_irqsave(&i915->mm.obj_lock, flags); i915->mm.shrink_count++; i915->mm.shrink_memory += obj->base.size; if (obj->mm.madv != I915_MADV_WILLNEED) list = &i915->mm.purge_list; else list = &i915->mm.shrink_list; list_add_tail(&obj->mm.link, list); atomic_set(&obj->mm.shrink_pin, 0); spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } } int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); int err; assert_object_held_shared(obj); if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { drm_dbg(&i915->drm, "Attempting to obtain a purgeable object\n"); return -EFAULT; } err = obj->ops->get_pages(obj); GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj)); return err; } /* Ensure that the associated pages are gathered from the backing storage * and pinned into our object. i915_gem_object_pin_pages() may be called * multiple times before they are released by a single call to * i915_gem_object_unpin_pages() - once the pages are no longer referenced * either as a result of memory pressure (reaping pages under the shrinker) * or as the object is itself released. */ int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) { int err; assert_object_held(obj); assert_object_held_shared(obj); if (unlikely(!i915_gem_object_has_pages(obj))) { GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); err = ____i915_gem_object_get_pages(obj); if (err) return err; smp_mb__before_atomic(); } atomic_inc(&obj->mm.pages_pin_count); return 0; } int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj) { struct i915_gem_ww_ctx ww; int err; i915_gem_ww_ctx_init(&ww, true); retry: err = i915_gem_object_lock(obj, &ww); if (!err) err = i915_gem_object_pin_pages(obj); if (err == -EDEADLK) { err = i915_gem_ww_ctx_backoff(&ww); if (!err) goto retry; } i915_gem_ww_ctx_fini(&ww); return err; } /* Immediately discard the backing storage */ int i915_gem_object_truncate(struct drm_i915_gem_object *obj) { if (obj->ops->truncate) return obj->ops->truncate(obj); return 0; } static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) { struct radix_tree_iter iter; void __rcu **slot; rcu_read_lock(); radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) radix_tree_delete(&obj->mm.get_page.radix, iter.index); radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0) radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index); rcu_read_unlock(); } static void unmap_object(struct drm_i915_gem_object *obj, void *ptr) { if (is_vmalloc_addr(ptr)) vunmap(ptr, obj->base.size); } static void flush_tlb_invalidate(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct intel_gt *gt; int id; for_each_gt(gt, i915, id) { if (!obj->mm.tlb[id]) continue; intel_gt_invalidate_tlb_full(gt, obj->mm.tlb[id]); obj->mm.tlb[id] = 0; } } struct sg_table * __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) { struct sg_table *pages; assert_object_held_shared(obj); pages = fetch_and_zero(&obj->mm.pages); if (IS_ERR_OR_NULL(pages)) return pages; if (i915_gem_object_is_volatile(obj)) obj->mm.madv = I915_MADV_WILLNEED; if (!i915_gem_object_has_self_managed_shrink_list(obj)) i915_gem_object_make_unshrinkable(obj); if (obj->mm.mapping) { unmap_object(obj, page_mask_bits(obj->mm.mapping)); obj->mm.mapping = NULL; } __i915_gem_object_reset_page_iter(obj); obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0; flush_tlb_invalidate(obj); return pages; } int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) { struct sg_table *pages; if (i915_gem_object_has_pinned_pages(obj)) return -EBUSY; /* May be called by shrinker from within get_pages() (on another bo) */ assert_object_held_shared(obj); i915_gem_object_release_mmap_offset(obj); /* * ->put_pages might need to allocate memory for the bit17 swizzle * array, hence protect them from being reaped by removing them from gtt * lists early. */ pages = __i915_gem_object_unset_pages(obj); /* * XXX Temporary hijinx to avoid updating all backends to handle * NULL pages. In the future, when we have more asynchronous * get_pages backends we should be better able to handle the * cancellation of the async task in a more uniform manner. */ if (!IS_ERR_OR_NULL(pages)) obj->ops->put_pages(obj, pages); return 0; } /* The 'mapping' part of i915_gem_object_pin_map() below */ static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj, enum i915_map_type type) { unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i; struct vm_page *stack[32], **pages = stack, *page; struct sgt_iter iter; pgprot_t pgprot; void *vaddr; switch (type) { default: MISSING_CASE(type); fallthrough; /* to use PAGE_KERNEL anyway */ case I915_MAP_WB: /* * On 32b, highmem using a finite set of indirect PTE (i.e. * vmap) to provide virtual mappings of the high pages. * As these are finite, map_new_virtual() must wait for some * other kmap() to finish when it runs out. If we map a large * number of objects, there is no method for it to tell us * to release the mappings, and we deadlock. * * However, if we make an explicit vmap of the page, that * uses a larger vmalloc arena, and also has the ability * to tell us to release unwanted mappings. Most importantly, * it will fail and propagate an error instead of waiting * forever. * * So if the page is beyond the 32b boundary, make an explicit * vmap. */ #ifdef notyet if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl))) return page_address(sg_page(obj->mm.pages->sgl)); #endif pgprot = PAGE_KERNEL; break; case I915_MAP_WC: pgprot = pgprot_writecombine(PAGE_KERNEL_IO); break; } if (n_pages > ARRAY_SIZE(stack)) { /* Too big for stack -- allocate temporary array instead */ pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL); if (!pages) return ERR_PTR(-ENOMEM); } i = 0; for_each_sgt_page(page, iter, obj->mm.pages) pages[i++] = page; vaddr = vmap(pages, n_pages, 0, pgprot); if (pages != stack) kvfree(pages); return vaddr ?: ERR_PTR(-ENOMEM); } static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj, enum i915_map_type type) { resource_size_t iomap = obj->mm.region->iomap.base - obj->mm.region->region.start; unsigned long n_pfn = obj->base.size >> PAGE_SHIFT; unsigned long stack[32], *pfns = stack, i; struct sgt_iter iter; dma_addr_t addr; void *vaddr; GEM_BUG_ON(type != I915_MAP_WC); if (n_pfn > ARRAY_SIZE(stack)) { /* Too big for stack -- allocate temporary array instead */ pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL); if (!pfns) return ERR_PTR(-ENOMEM); } i = 0; for_each_sgt_daddr(addr, iter, obj->mm.pages) pfns[i++] = (iomap + addr) >> PAGE_SHIFT; vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO)); if (pfns != stack) kvfree(pfns); return vaddr ?: ERR_PTR(-ENOMEM); } /* get, pin, and map the pages of the object into kernel space */ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, enum i915_map_type type) { enum i915_map_type has_type; bool pinned; void *ptr; int err; if (!i915_gem_object_has_struct_page(obj) && !i915_gem_object_has_iomem(obj)) return ERR_PTR(-ENXIO); if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY)) return ERR_PTR(-EINVAL); assert_object_held(obj); pinned = !(type & I915_MAP_OVERRIDE); type &= ~I915_MAP_OVERRIDE; if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { if (unlikely(!i915_gem_object_has_pages(obj))) { GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); err = ____i915_gem_object_get_pages(obj); if (err) return ERR_PTR(err); smp_mb__before_atomic(); } atomic_inc(&obj->mm.pages_pin_count); pinned = false; } GEM_BUG_ON(!i915_gem_object_has_pages(obj)); /* * For discrete our CPU mappings needs to be consistent in order to * function correctly on !x86. When mapping things through TTM, we use * the same rules to determine the caching type. * * The caching rules, starting from DG1: * * - If the object can be placed in device local-memory, then the * pages should be allocated and mapped as write-combined only. * * - Everything else is always allocated and mapped as write-back, * with the guarantee that everything is also coherent with the * GPU. * * Internal users of lmem are already expected to get this right, so no * fudging needed there. */ if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) { if (type != I915_MAP_WC && !obj->mm.n_placements) { ptr = ERR_PTR(-ENODEV); goto err_unpin; } type = I915_MAP_WC; } else if (IS_DGFX(to_i915(obj->base.dev))) { type = I915_MAP_WB; } ptr = page_unpack_bits(obj->mm.mapping, &has_type); if (ptr && has_type != type) { if (pinned) { ptr = ERR_PTR(-EBUSY); goto err_unpin; } unmap_object(obj, ptr); ptr = obj->mm.mapping = NULL; } if (!ptr) { err = i915_gem_object_wait_moving_fence(obj, true); if (err) { ptr = ERR_PTR(err); goto err_unpin; } if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled())) ptr = ERR_PTR(-ENODEV); else if (i915_gem_object_has_struct_page(obj)) ptr = i915_gem_object_map_page(obj, type); else ptr = i915_gem_object_map_pfn(obj, type); if (IS_ERR(ptr)) goto err_unpin; obj->mm.mapping = page_pack_bits(ptr, type); } return ptr; err_unpin: atomic_dec(&obj->mm.pages_pin_count); return ptr; } void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj, enum i915_map_type type) { void *ret; i915_gem_object_lock(obj, NULL); ret = i915_gem_object_pin_map(obj, type); i915_gem_object_unlock(obj); return ret; } void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, unsigned long offset, unsigned long size) { enum i915_map_type has_type; void *ptr; GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), offset, size, obj->base.size)); wmb(); /* let all previous writes be visible to coherent partners */ obj->mm.dirty = true; if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) return; ptr = page_unpack_bits(obj->mm.mapping, &has_type); if (has_type == I915_MAP_WC) return; drm_clflush_virt_range(ptr + offset, size); if (size == obj->base.size) { obj->write_domain &= ~I915_GEM_DOMAIN_CPU; obj->cache_dirty = false; } } void __i915_gem_object_release_map(struct drm_i915_gem_object *obj) { GEM_BUG_ON(!obj->mm.mapping); /* * We allow removing the mapping from underneath pinned pages! * * Furthermore, since this is an unsafe operation reserved only * for construction time manipulation, we ignore locking prudence. */ unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping))); i915_gem_object_unpin_map(obj); } struct scatterlist * __i915_gem_object_page_iter_get_sg(struct drm_i915_gem_object *obj, struct i915_gem_object_page_iter *iter, pgoff_t n, unsigned int *offset) { const bool dma = iter == &obj->mm.get_dma_page || iter == &obj->ttm.get_io_page; unsigned int idx, count; struct scatterlist *sg; might_sleep(); GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); if (!i915_gem_object_has_pinned_pages(obj)) assert_object_held(obj); /* As we iterate forward through the sg, we record each entry in a * radixtree for quick repeated (backwards) lookups. If we have seen * this index previously, we will have an entry for it. * * Initial lookup is O(N), but this is amortized to O(1) for * sequential page access (where each new request is consecutive * to the previous one). Repeated lookups are O(lg(obj->base.size)), * i.e. O(1) with a large constant! */ if (n < READ_ONCE(iter->sg_idx)) goto lookup; mutex_lock(&iter->lock); /* We prefer to reuse the last sg so that repeated lookup of this * (or the subsequent) sg are fast - comparing against the last * sg is faster than going through the radixtree. */ sg = iter->sg_pos; idx = iter->sg_idx; count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); while (idx + count <= n) { void *entry; unsigned long i; int ret; /* If we cannot allocate and insert this entry, or the * individual pages from this range, cancel updating the * sg_idx so that on this lookup we are forced to linearly * scan onwards, but on future lookups we will try the * insertion again (in which case we need to be careful of * the error return reporting that we have already inserted * this index). */ ret = radix_tree_insert(&iter->radix, idx, sg); if (ret && ret != -EEXIST) goto scan; entry = xa_mk_value(idx); for (i = 1; i < count; i++) { ret = radix_tree_insert(&iter->radix, idx + i, entry); if (ret && ret != -EEXIST) goto scan; } idx += count; sg = ____sg_next(sg); count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); } scan: iter->sg_pos = sg; iter->sg_idx = idx; mutex_unlock(&iter->lock); if (unlikely(n < idx)) /* insertion completed by another thread */ goto lookup; /* In case we failed to insert the entry into the radixtree, we need * to look beyond the current sg. */ while (idx + count <= n) { idx += count; sg = ____sg_next(sg); count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg); } *offset = n - idx; return sg; lookup: rcu_read_lock(); sg = radix_tree_lookup(&iter->radix, n); GEM_BUG_ON(!sg); /* If this index is in the middle of multi-page sg entry, * the radix tree will contain a value entry that points * to the start of that range. We will return the pointer to * the base page and the offset of this page within the * sg entry's range. */ *offset = 0; if (unlikely(xa_is_value(sg))) { unsigned long base = xa_to_value(sg); sg = radix_tree_lookup(&iter->radix, base); GEM_BUG_ON(!sg); *offset = n - base; } rcu_read_unlock(); return sg; } struct vm_page * __i915_gem_object_get_page(struct drm_i915_gem_object *obj, pgoff_t n) { struct scatterlist *sg; unsigned int offset; GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); sg = i915_gem_object_get_sg(obj, n, &offset); return nth_page(sg_page(sg), offset); } /* Like i915_gem_object_get_page(), but mark the returned page dirty */ struct vm_page * __i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, pgoff_t n) { struct vm_page *page; page = i915_gem_object_get_page(obj, n); if (!obj->mm.dirty) set_page_dirty(page); return page; } dma_addr_t __i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, pgoff_t n, unsigned int *len) { struct scatterlist *sg; unsigned int offset; sg = i915_gem_object_get_sg_dma(obj, n, &offset); if (len) *len = sg_dma_len(sg) - (offset << PAGE_SHIFT); return sg_dma_address(sg) + (offset << PAGE_SHIFT); } dma_addr_t __i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, pgoff_t n) { return i915_gem_object_get_dma_address_len(obj, n, NULL); }