/* * Copyright (c) Red Hat Inc. * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sub license, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * * Authors: Dave Airlie * Jerome Glisse * Pauli Nieminen */ /* simple list based uncached page pool * - Pool collects resently freed pages for reuse * - Use page->lru to keep a free list * - doesn't track currently in use pages */ #define pr_fmt(fmt) "[TTM] " fmt #include #include //#include //#include #include #include #include /* for seq_printf */ #include #include //#include #include #include #ifdef TTM_HAS_AGP #include #endif #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) #define SMALL_ALLOCATION 16 #define FREE_ALL_PAGES (~0U) /* times are in msecs */ #define PAGE_FREE_INTERVAL 1000 #define pr_err(fmt, ...) \ printk(KERN_ERR pr_fmt(fmt), ##__VA_ARGS__) #if 0 /** * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. * * @lock: Protects the shared pool from concurrnet access. Must be used with * irqsave/irqrestore variants because pool allocator maybe called from * delayed work. * @fill_lock: Prevent concurrent calls to fill. * @list: Pool of free uc/wc pages for fast reuse. * @gfp_flags: Flags to pass for alloc_page. * @npages: Number of pages in pool. */ struct ttm_page_pool { spinlock_t lock; bool fill_lock; struct list_head list; gfp_t gfp_flags; unsigned npages; char *name; unsigned long nfrees; unsigned long nrefills; }; /** * Limits for the pool. They are handled without locks because only place where * they may change is in sysfs store. They won't have immediate effect anyway * so forcing serialization to access them is pointless. */ struct ttm_pool_opts { unsigned alloc_size; unsigned max_size; unsigned small; }; #define NUM_POOLS 4 /** * struct ttm_pool_manager - Holds memory pools for fst allocation * * Manager is read only object for pool code so it doesn't need locking. * * @free_interval: minimum number of jiffies between freeing pages from pool. * @page_alloc_inited: reference counting for pool allocation. * @work: Work that is used to shrink the pool. Work is only run when there is * some pages to free. * @small_allocation: Limit in number of pages what is small allocation. * * @pools: All pool objects in use. **/ struct ttm_pool_manager { struct kobject kobj; struct shrinker mm_shrink; struct ttm_pool_opts options; union { struct ttm_page_pool pools[NUM_POOLS]; struct { struct ttm_page_pool wc_pool; struct ttm_page_pool uc_pool; struct ttm_page_pool wc_pool_dma32; struct ttm_page_pool uc_pool_dma32; } ; }; }; static struct attribute ttm_page_pool_max = { .name = "pool_max_size", .mode = S_IRUGO | S_IWUSR }; static struct attribute ttm_page_pool_small = { .name = "pool_small_allocation", .mode = S_IRUGO | S_IWUSR }; static struct attribute ttm_page_pool_alloc_size = { .name = "pool_allocation_size", .mode = S_IRUGO | S_IWUSR }; static struct attribute *ttm_pool_attrs[] = { &ttm_page_pool_max, &ttm_page_pool_small, &ttm_page_pool_alloc_size, NULL }; static void ttm_pool_kobj_release(struct kobject *kobj) { struct ttm_pool_manager *m = container_of(kobj, struct ttm_pool_manager, kobj); kfree(m); } static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, const char *buffer, size_t size) { struct ttm_pool_manager *m = container_of(kobj, struct ttm_pool_manager, kobj); int chars; unsigned val; chars = sscanf(buffer, "%u", &val); if (chars == 0) return size; /* Convert kb to number of pages */ val = val / (PAGE_SIZE >> 10); if (attr == &ttm_page_pool_max) m->options.max_size = val; else if (attr == &ttm_page_pool_small) m->options.small = val; else if (attr == &ttm_page_pool_alloc_size) { if (val > NUM_PAGES_TO_ALLOC*8) { pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); return size; } else if (val > NUM_PAGES_TO_ALLOC) { pr_warn("Setting allocation size to larger than %lu is not recommended\n", NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); } m->options.alloc_size = val; } return size; } static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, char *buffer) { struct ttm_pool_manager *m = container_of(kobj, struct ttm_pool_manager, kobj); unsigned val = 0; if (attr == &ttm_page_pool_max) val = m->options.max_size; else if (attr == &ttm_page_pool_small) val = m->options.small; else if (attr == &ttm_page_pool_alloc_size) val = m->options.alloc_size; val = val * (PAGE_SIZE >> 10); return snprintf(buffer, PAGE_SIZE, "%u\n", val); } static const struct sysfs_ops ttm_pool_sysfs_ops = { .show = &ttm_pool_show, .store = &ttm_pool_store, }; static struct kobj_type ttm_pool_kobj_type = { .release = &ttm_pool_kobj_release, .sysfs_ops = &ttm_pool_sysfs_ops, .default_attrs = ttm_pool_attrs, }; static struct ttm_pool_manager *_manager; #ifndef CONFIG_X86 static int set_pages_array_wb(struct page **pages, int addrinarray) { #ifdef TTM_HAS_AGP int i; for (i = 0; i < addrinarray; i++) unmap_page_from_agp(pages[i]); #endif return 0; } static int set_pages_array_wc(struct page **pages, int addrinarray) { #ifdef TTM_HAS_AGP int i; for (i = 0; i < addrinarray; i++) map_page_into_agp(pages[i]); #endif return 0; } static int set_pages_array_uc(struct page **pages, int addrinarray) { #ifdef TTM_HAS_AGP int i; for (i = 0; i < addrinarray; i++) map_page_into_agp(pages[i]); #endif return 0; } #endif /** * Select the right pool or requested caching state and ttm flags. */ static struct ttm_page_pool *ttm_get_pool(int flags, enum ttm_caching_state cstate) { int pool_index; if (cstate == tt_cached) return NULL; if (cstate == tt_wc) pool_index = 0x0; else pool_index = 0x1; if (flags & TTM_PAGE_FLAG_DMA32) pool_index |= 0x2; return &_manager->pools[pool_index]; } /* set memory back to wb and free the pages. */ static void ttm_pages_put(struct page *pages[], unsigned npages) { unsigned i; if (set_pages_array_wb(pages, npages)) pr_err("Failed to set %d pages to wb!\n", npages); for (i = 0; i < npages; ++i) __free_page(pages[i]); } static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, unsigned freed_pages) { pool->npages -= freed_pages; pool->nfrees += freed_pages; } /** * Free pages from pool. * * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC * number of pages in one go. * * @pool: to free the pages from * @free_all: If set to true will free all pages in pool **/ static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) { unsigned long irq_flags; struct page *p; struct page **pages_to_free; unsigned freed_pages = 0, npages_to_free = nr_free; if (NUM_PAGES_TO_ALLOC < nr_free) npages_to_free = NUM_PAGES_TO_ALLOC; pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), GFP_KERNEL); if (!pages_to_free) { pr_err("Failed to allocate memory for pool free operation\n"); return 0; } restart: spin_lock_irqsave(&pool->lock, irq_flags); list_for_each_entry_reverse(p, &pool->list, lru) { if (freed_pages >= npages_to_free) break; pages_to_free[freed_pages++] = p; /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ if (freed_pages >= NUM_PAGES_TO_ALLOC) { /* remove range of pages from the pool */ __list_del(p->lru.prev, &pool->list); ttm_pool_update_free_locked(pool, freed_pages); /** * Because changing page caching is costly * we unlock the pool to prevent stalling. */ spin_unlock_irqrestore(&pool->lock, irq_flags); ttm_pages_put(pages_to_free, freed_pages); if (likely(nr_free != FREE_ALL_PAGES)) nr_free -= freed_pages; if (NUM_PAGES_TO_ALLOC >= nr_free) npages_to_free = nr_free; else npages_to_free = NUM_PAGES_TO_ALLOC; freed_pages = 0; /* free all so restart the processing */ if (nr_free) goto restart; /* Not allowed to fall through or break because * following context is inside spinlock while we are * outside here. */ goto out; } } /* remove range of pages from the pool */ if (freed_pages) { __list_del(&p->lru, &pool->list); ttm_pool_update_free_locked(pool, freed_pages); nr_free -= freed_pages; } spin_unlock_irqrestore(&pool->lock, irq_flags); if (freed_pages) ttm_pages_put(pages_to_free, freed_pages); out: kfree(pages_to_free); return nr_free; } /** * Callback for mm to request pool to reduce number of page held. * * XXX: (dchinner) Deadlock warning! * * ttm_page_pool_free() does memory allocation using GFP_KERNEL. that means * this can deadlock when called a sc->gfp_mask that is not equal to * GFP_KERNEL. * * This code is crying out for a shrinker per pool.... */ static unsigned long ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { static atomic_t start_pool = ATOMIC_INIT(0); unsigned i; unsigned pool_offset = atomic_add_return(1, &start_pool); struct ttm_page_pool *pool; int shrink_pages = sc->nr_to_scan; unsigned long freed = 0; pool_offset = pool_offset % NUM_POOLS; /* select start pool in round robin fashion */ for (i = 0; i < NUM_POOLS; ++i) { unsigned nr_free = shrink_pages; if (shrink_pages == 0) break; pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; shrink_pages = ttm_page_pool_free(pool, nr_free); freed += nr_free - shrink_pages; } return freed; } static unsigned long ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { unsigned i; unsigned long count = 0; for (i = 0; i < NUM_POOLS; ++i) count += _manager->pools[i].npages; return count; } static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) { manager->mm_shrink.count_objects = ttm_pool_shrink_count; manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; manager->mm_shrink.seeks = 1; register_shrinker(&manager->mm_shrink); } static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) { unregister_shrinker(&manager->mm_shrink); } static int ttm_set_pages_caching(struct page **pages, enum ttm_caching_state cstate, unsigned cpages) { int r = 0; /* Set page caching */ switch (cstate) { case tt_uncached: r = set_pages_array_uc(pages, cpages); if (r) pr_err("Failed to set %d pages to uc!\n", cpages); break; case tt_wc: r = set_pages_array_wc(pages, cpages); if (r) pr_err("Failed to set %d pages to wc!\n", cpages); break; default: break; } return r; } /** * Free pages the pages that failed to change the caching state. If there is * any pages that have changed their caching state already put them to the * pool. */ static void ttm_handle_caching_state_failure(struct list_head *pages, int ttm_flags, enum ttm_caching_state cstate, struct page **failed_pages, unsigned cpages) { unsigned i; /* Failed pages have to be freed */ for (i = 0; i < cpages; ++i) { list_del(&failed_pages[i]->lru); __free_page(failed_pages[i]); } } /** * Allocate new pages with correct caching. * * This function is reentrant if caller updates count depending on number of * pages returned in pages array. */ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, int ttm_flags, enum ttm_caching_state cstate, unsigned count) { struct page **caching_array; struct page *p; int r = 0; unsigned i, cpages; unsigned max_cpages = min(count, (unsigned)(PAGE_SIZE/sizeof(struct page *))); /* allocate array for page caching change */ caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); if (!caching_array) { pr_err("Unable to allocate table for new pages\n"); return -ENOMEM; } for (i = 0, cpages = 0; i < count; ++i) { p = alloc_page(gfp_flags); if (!p) { pr_err("Unable to get page %u\n", i); /* store already allocated pages in the pool after * setting the caching state */ if (cpages) { r = ttm_set_pages_caching(caching_array, cstate, cpages); if (r) ttm_handle_caching_state_failure(pages, ttm_flags, cstate, caching_array, cpages); } r = -ENOMEM; goto out; } #ifdef CONFIG_HIGHMEM /* gfp flags of highmem page should never be dma32 so we * we should be fine in such case */ if (!PageHighMem(p)) #endif { caching_array[cpages++] = p; if (cpages == max_cpages) { r = ttm_set_pages_caching(caching_array, cstate, cpages); if (r) { ttm_handle_caching_state_failure(pages, ttm_flags, cstate, caching_array, cpages); goto out; } cpages = 0; } } list_add(&p->lru, pages); } if (cpages) { r = ttm_set_pages_caching(caching_array, cstate, cpages); if (r) ttm_handle_caching_state_failure(pages, ttm_flags, cstate, caching_array, cpages); } out: kfree(caching_array); return r; } /** * Fill the given pool if there aren't enough pages and the requested number of * pages is small. */ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, enum ttm_caching_state cstate, unsigned count, unsigned long *irq_flags) { struct page *p; int r; unsigned cpages = 0; /** * Only allow one pool fill operation at a time. * If pool doesn't have enough pages for the allocation new pages are * allocated from outside of pool. */ if (pool->fill_lock) return; pool->fill_lock = true; /* If allocation request is small and there are not enough * pages in a pool we fill the pool up first. */ if (count < _manager->options.small && count > pool->npages) { struct list_head new_pages; unsigned alloc_size = _manager->options.alloc_size; /** * Can't change page caching if in irqsave context. We have to * drop the pool->lock. */ spin_unlock_irqrestore(&pool->lock, *irq_flags); INIT_LIST_HEAD(&new_pages); r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, cstate, alloc_size); spin_lock_irqsave(&pool->lock, *irq_flags); if (!r) { list_splice(&new_pages, &pool->list); ++pool->nrefills; pool->npages += alloc_size; } else { pr_err("Failed to fill pool (%p)\n", pool); /* If we have any pages left put them to the pool. */ list_for_each_entry(p, &pool->list, lru) { ++cpages; } list_splice(&new_pages, &pool->list); pool->npages += cpages; } } pool->fill_lock = false; } /** * Cut 'count' number of pages from the pool and put them on the return list. * * @return count of pages still required to fulfill the request. */ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, struct list_head *pages, int ttm_flags, enum ttm_caching_state cstate, unsigned count) { unsigned long irq_flags; struct list_head *p; unsigned i; spin_lock_irqsave(&pool->lock, irq_flags); ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); if (count >= pool->npages) { /* take all pages from the pool */ list_splice_init(&pool->list, pages); count -= pool->npages; pool->npages = 0; goto out; } /* find the last pages to include for requested number of pages. Split * pool to begin and halve it to reduce search space. */ if (count <= pool->npages/2) { i = 0; list_for_each(p, &pool->list) { if (++i == count) break; } } else { i = pool->npages + 1; list_for_each_prev(p, &pool->list) { if (--i == count) break; } } /* Cut 'count' number of pages from the pool */ list_cut_position(pages, &pool->list, p); pool->npages -= count; count = 0; out: spin_unlock_irqrestore(&pool->lock, irq_flags); return count; } #endif /* Put all pages in pages list to correct pool to wait for reuse */ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, enum ttm_caching_state cstate) { unsigned long irq_flags; // struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); unsigned i; for (i = 0; i < npages; i++) { if (pages[i]) { // if (page_count(pages[i]) != 1) // pr_err("Erroneous page count. Leaking pages.\n"); FreePage(pages[i]); pages[i] = NULL; } } return; #if 0 if (pool == NULL) { /* No pool for this memory type so free the pages */ for (i = 0; i < npages; i++) { if (pages[i]) { if (page_count(pages[i]) != 1) pr_err("Erroneous page count. Leaking pages.\n"); __free_page(pages[i]); pages[i] = NULL; } } return; } spin_lock_irqsave(&pool->lock, irq_flags); for (i = 0; i < npages; i++) { if (pages[i]) { if (page_count(pages[i]) != 1) pr_err("Erroneous page count. Leaking pages.\n"); list_add_tail(&pages[i]->lru, &pool->list); pages[i] = NULL; pool->npages++; } } /* Check that we don't go over the pool limit */ npages = 0; if (pool->npages > _manager->options.max_size) { npages = pool->npages - _manager->options.max_size; /* free at least NUM_PAGES_TO_ALLOC number of pages * to reduce calls to set_memory_wb */ if (npages < NUM_PAGES_TO_ALLOC) npages = NUM_PAGES_TO_ALLOC; } spin_unlock_irqrestore(&pool->lock, irq_flags); if (npages) ttm_page_pool_free(pool, npages); #endif } /* * On success pages list will hold count number of correctly * cached pages. */ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, enum ttm_caching_state cstate) { // struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); struct list_head plist; struct page *p = NULL; // gfp_t gfp_flags = GFP_USER; unsigned count; int r; for (r = 0; r < npages; ++r) { p = AllocPage(); if (!p) { pr_err("Unable to allocate page\n"); return -ENOMEM; } pages[r] = p; } return 0; #if 0 /* set zero flag for page allocation if required */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) gfp_flags |= __GFP_ZERO; /* No pool for cached pages */ if (pool == NULL) { if (flags & TTM_PAGE_FLAG_DMA32) gfp_flags |= GFP_DMA32; else gfp_flags |= GFP_HIGHUSER; for (r = 0; r < npages; ++r) { p = alloc_page(gfp_flags); if (!p) { pr_err("Unable to allocate page\n"); return -ENOMEM; } pages[r] = p; } return 0; } /* combine zero flag to pool flags */ gfp_flags |= pool->gfp_flags; /* First we take pages from the pool */ INIT_LIST_HEAD(&plist); npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); count = 0; list_for_each_entry(p, &plist, lru) { pages[count++] = p; } /* clear the pages coming from the pool if requested */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { list_for_each_entry(p, &plist, lru) { if (PageHighMem(p)) clear_highpage(p); else clear_page(page_address(p)); } } /* If pool didn't have enough pages allocate new one. */ if (npages > 0) { /* ttm_alloc_new_pages doesn't reference pool so we can run * multiple requests in parallel. **/ INIT_LIST_HEAD(&plist); r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages); list_for_each_entry(p, &plist, lru) { pages[count++] = p; } if (r) { /* If there is any pages in the list put them back to * the pool. */ pr_err("Failed to allocate extra pages for large request\n"); ttm_put_pages(pages, count, flags, cstate); return r; } } #endif return 0; } #if 0 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, char *name) { spin_lock_init(&pool->lock); pool->fill_lock = false; INIT_LIST_HEAD(&pool->list); pool->npages = pool->nfrees = 0; pool->gfp_flags = flags; pool->name = name; } int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) { int ret; WARN_ON(_manager); pr_info("Initializing pool allocator\n"); _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); ttm_page_pool_init_locked(&_manager->wc_pool_dma32, GFP_USER | GFP_DMA32, "wc dma"); ttm_page_pool_init_locked(&_manager->uc_pool_dma32, GFP_USER | GFP_DMA32, "uc dma"); _manager->options.max_size = max_pages; _manager->options.small = SMALL_ALLOCATION; _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, &glob->kobj, "pool"); if (unlikely(ret != 0)) { kobject_put(&_manager->kobj); _manager = NULL; return ret; } ttm_pool_mm_shrink_init(_manager); return 0; } void ttm_page_alloc_fini(void) { int i; pr_info("Finalizing pool allocator\n"); ttm_pool_mm_shrink_fini(_manager); for (i = 0; i < NUM_POOLS; ++i) ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); kobject_put(&_manager->kobj); _manager = NULL; } #endif int ttm_pool_populate(struct ttm_tt *ttm) { struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; unsigned i; int ret; if (ttm->state != tt_unpopulated) return 0; for (i = 0; i < ttm->num_pages; ++i) { ret = ttm_get_pages(&ttm->pages[i], 1, ttm->page_flags, ttm->caching_state); if (ret != 0) { ttm_pool_unpopulate(ttm); return -ENOMEM; } } ttm->state = tt_unbound; return 0; } EXPORT_SYMBOL(ttm_pool_populate); void ttm_pool_unpopulate(struct ttm_tt *ttm) { unsigned i; for (i = 0; i < ttm->num_pages; ++i) { if (ttm->pages[i]) { ttm_mem_global_free_page(ttm->glob->mem_glob, ttm->pages[i]); ttm_put_pages(&ttm->pages[i], 1, ttm->page_flags, ttm->caching_state); } } ttm->state = tt_unpopulated; } EXPORT_SYMBOL(ttm_pool_unpopulate);